Qualify Windows system delegation and crash-safe mirror adapter on native VM
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -0,0 +1,414 @@
|
||||
//! Fixed, preprovisioned confidential delegation journal. Never log request bytes.
|
||||
use crate::helper_ipc::{self as ipc, FileIdentity, PinnedFile};
|
||||
use anyhow::{Result, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rmm_agent_core::{
|
||||
auth::Request,
|
||||
helper_reconcile::{NoDispatchObservation, NoStartAnchor},
|
||||
};
|
||||
use rmm_domain::jobs::{ExecutionPermitPayload, hash};
|
||||
use rusqlite::{Connection, OpenFlags, OptionalExtension, TransactionBehavior, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{path::Path, time::Duration};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const ROOT: &str = r"C:\ProgramData\RustDiscoverySystem\agent-intents";
|
||||
pub const DB: &str = r"C:\ProgramData\RustDiscoverySystem\agent-intents\system-intents.sqlite";
|
||||
const ACTOR_RW: u32 = 0x0012_019f;
|
||||
const SCHEMA:&str="CREATE TABLE IF NOT EXISTS system_intent_meta(id INTEGER PRIMARY KEY CHECK(id=1),file_identity BLOB NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS system_intents(id TEXT PRIMARY KEY,job TEXT NOT NULL,attempt TEXT NOT NULL,grant_id TEXT NOT NULL UNIQUE,body BLOB NOT NULL CHECK(length(body)<=196608),digest TEXT NOT NULL,phase TEXT NOT NULL CHECK(phase IN ('prepared','known_no_dispatch','may_have_dispatched')),UNIQUE(job,attempt));
|
||||
CREATE TABLE IF NOT EXISTS system_intent_mirrors(id TEXT PRIMARY KEY,outcome_sha256 TEXT NOT NULL,mirror_observed_at TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS system_intent_denial_resumes(id TEXT NOT NULL,agent_instance TEXT NOT NULL,observed_at TEXT NOT NULL,PRIMARY KEY(id,agent_instance));
|
||||
CREATE TRIGGER IF NOT EXISTS system_intents_no_delete BEFORE DELETE ON system_intents BEGIN SELECT RAISE(ABORT,'SYSTEM_INTENT_IMMUTABLE');END;
|
||||
CREATE TRIGGER IF NOT EXISTS system_intents_monotonic BEFORE UPDATE ON system_intents WHEN NEW.id!=OLD.id OR NEW.job!=OLD.job OR NEW.attempt!=OLD.attempt OR NEW.grant_id!=OLD.grant_id OR NEW.body!=OLD.body OR NEW.digest!=OLD.digest OR OLD.phase!='prepared' OR NEW.phase NOT IN ('known_no_dispatch','may_have_dispatched') BEGIN SELECT RAISE(ABORT,'SYSTEM_INTENT_IMMUTABLE');END;
|
||||
CREATE TRIGGER IF NOT EXISTS system_mirrors_no_delete BEFORE DELETE ON system_intent_mirrors BEGIN SELECT RAISE(ABORT,'SYSTEM_MIRROR_IMMUTABLE');END;
|
||||
CREATE TRIGGER IF NOT EXISTS system_mirrors_no_update BEFORE UPDATE ON system_intent_mirrors BEGIN SELECT RAISE(ABORT,'SYSTEM_MIRROR_IMMUTABLE');END;
|
||||
CREATE TRIGGER IF NOT EXISTS system_denial_resumes_no_delete BEFORE DELETE ON system_intent_denial_resumes BEGIN SELECT RAISE(ABORT,'SYSTEM_DENIAL_RESUME_IMMUTABLE');END;
|
||||
CREATE TRIGGER IF NOT EXISTS system_denial_resumes_no_update BEFORE UPDATE ON system_intent_denial_resumes BEGIN SELECT RAISE(ABORT,'SYSTEM_DENIAL_RESUME_IMMUTABLE');END;";
|
||||
const CLAIM_RESUME: &str = "INSERT OR IGNORE INTO system_intent_denial_resumes(id,agent_instance,observed_at) VALUES(?1,?2,?3)";
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Intent {
|
||||
pub schema_version: u32,
|
||||
pub id: Uuid,
|
||||
pub request: Request,
|
||||
pub anchor: NoStartAnchor,
|
||||
pub original_boot_id: Uuid,
|
||||
pub agent_instance: Uuid,
|
||||
pub connection_generation: u64,
|
||||
pub recorded_at: DateTime<Utc>,
|
||||
pub policy_sha256: String,
|
||||
pub helper_binary_sha256: String,
|
||||
}
|
||||
impl Intent {
|
||||
pub fn digest(&self) -> Result<String> {
|
||||
Ok(hash(&serde_json::to_vec(self)?))
|
||||
}
|
||||
pub fn permit(&self) -> Result<ExecutionPermitPayload> {
|
||||
Ok(serde_json::from_slice(&self.request.permit_bytes)?)
|
||||
}
|
||||
fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.schema_version == 1
|
||||
&& !self.id.is_nil()
|
||||
&& !self.agent_instance.is_nil()
|
||||
&& !self.original_boot_id.is_nil()
|
||||
&& self.original_boot_id == self.anchor.boot_id()
|
||||
&& self.connection_generation == self.anchor.connection_generation()
|
||||
&& ipc::digest(&self.policy_sha256)
|
||||
&& ipc::digest(&self.helper_binary_sha256)
|
||||
&& serde_json::to_vec(self)?.len() <= 196608,
|
||||
"SYSTEM_INTENT_BINDING"
|
||||
);
|
||||
let p = self.permit()?;
|
||||
ensure!(
|
||||
p.execution_identity == "system"
|
||||
&& p.authorization_id == self.request.authorization_id
|
||||
&& p.plan_hash == self.request.plan_hash
|
||||
&& p.job_payload_hash == hash(&self.request.job_bytes),
|
||||
"SYSTEM_INTENT_REQUEST_BINDING"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub enum Phase {
|
||||
Prepared,
|
||||
KnownNoDispatch,
|
||||
MayHaveDispatched,
|
||||
}
|
||||
impl Phase {
|
||||
fn parse(s: &str) -> Result<Self> {
|
||||
match s {
|
||||
"prepared" => Ok(Self::Prepared),
|
||||
"known_no_dispatch" => Ok(Self::KnownNoDispatch),
|
||||
"may_have_dispatched" => Ok(Self::MayHaveDispatched),
|
||||
_ => anyhow::bail!("SYSTEM_INTENT_PHASE"),
|
||||
}
|
||||
}
|
||||
fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Prepared => "prepared",
|
||||
Self::KnownNoDispatch => "known_no_dispatch",
|
||||
Self::MayHaveDispatched => "may_have_dispatched",
|
||||
}
|
||||
}
|
||||
}
|
||||
fn advance(db: &mut Connection, id: Uuid, desired: Phase) -> Result<(usize, Phase)> {
|
||||
let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let changed = tx.execute(
|
||||
"UPDATE system_intents SET phase=?2 WHERE id=?1 AND phase='prepared'",
|
||||
params![id.to_string(), desired.name()],
|
||||
)?;
|
||||
let phase: String = tx.query_row(
|
||||
"SELECT phase FROM system_intents WHERE id=?1",
|
||||
[id.to_string()],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok((changed, Phase::parse(&phase)?))
|
||||
}
|
||||
pub struct Store {
|
||||
db: Connection,
|
||||
files: Vec<PinnedFile>,
|
||||
identity: FileIdentity,
|
||||
}
|
||||
impl Store {
|
||||
/// Bootstrap, never LocalService, creates both fixed leaves and their ACLs.
|
||||
/// PERSIST retains the precreated rollback journal, requiring no Actor delete
|
||||
/// or directory-create rights. A hot journal on the read-only side refuses.
|
||||
pub fn open(write: bool) -> Result<Self> {
|
||||
let actor = ipc::service_sid(crate::SERVICE_NAME)?;
|
||||
let root =
|
||||
PinnedFile::open_read_shared(Path::new(ROOT), true, std::slice::from_ref(&actor), &[])?;
|
||||
let mut files = vec![root];
|
||||
for path in [DB, &format!("{DB}-journal")] {
|
||||
let mut file = if write {
|
||||
PinnedFile::open(
|
||||
Path::new(path),
|
||||
false,
|
||||
&[],
|
||||
std::slice::from_ref(&actor),
|
||||
true,
|
||||
)?
|
||||
} else {
|
||||
PinnedFile::open_read_shared(
|
||||
Path::new(path),
|
||||
false,
|
||||
&[],
|
||||
std::slice::from_ref(&actor),
|
||||
)?
|
||||
};
|
||||
file.limit_writer_rights(ACTOR_RW)?;
|
||||
files.push(file);
|
||||
}
|
||||
let identity = files[1].identity;
|
||||
let flags = if write {
|
||||
OpenFlags::SQLITE_OPEN_READ_WRITE
|
||||
} else {
|
||||
OpenFlags::SQLITE_OPEN_READ_ONLY
|
||||
};
|
||||
let db = Connection::open_with_flags(DB, flags)?;
|
||||
db.busy_timeout(Duration::from_millis(250))?;
|
||||
if write {
|
||||
rmm_agent_core::ledger::bounded_journal_capacity(&db)?;
|
||||
let mode: String = db.query_row("PRAGMA journal_mode=PERSIST", [], |r| r.get(0))?;
|
||||
ensure!(mode == "persist", "SYSTEM_INTENT_JOURNAL_MODE");
|
||||
db.execute_batch("PRAGMA synchronous=FULL;PRAGMA temp_store=MEMORY;PRAGMA journal_size_limit=1048576;")?;
|
||||
db.execute_batch(SCHEMA)?;
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO system_intent_meta VALUES(1,?1)",
|
||||
[serde_json::to_vec(&identity)?],
|
||||
)?;
|
||||
}
|
||||
let stored: Vec<u8> = db.query_row(
|
||||
"SELECT file_identity FROM system_intent_meta WHERE id=1",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
stored == serde_json::to_vec(&identity)?,
|
||||
"SYSTEM_INTENT_FILE_REPLACED"
|
||||
);
|
||||
ensure!(
|
||||
db.query_row("PRAGMA integrity_check", [], |r| r.get::<_, String>(0))? == "ok",
|
||||
"SYSTEM_INTENT_INTEGRITY"
|
||||
);
|
||||
let mut value = Self {
|
||||
db,
|
||||
files,
|
||||
identity,
|
||||
};
|
||||
value.check()?;
|
||||
Ok(value)
|
||||
}
|
||||
pub fn check(&mut self) -> Result<()> {
|
||||
for file in &mut self.files {
|
||||
file.check()?;
|
||||
}
|
||||
ensure!(
|
||||
ipc::file_identity(&self.files[1].file)? == self.identity,
|
||||
"SYSTEM_INTENT_FILE_REPLACED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub fn prepare(&mut self, intent: &Intent) -> Result<()> {
|
||||
self.check()?;
|
||||
intent.validate()?;
|
||||
let tx = self
|
||||
.db
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let count: u64 = tx.query_row("SELECT count(*) FROM system_intents", [], |r| r.get(0))?;
|
||||
ensure!(count < 10000, "SYSTEM_INTENT_CAPACITY_BLOCKED");
|
||||
let busy:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM system_intents i WHERE NOT EXISTS(SELECT 1 FROM system_intent_mirrors m WHERE m.id=i.id))",[],|r|r.get(0))?;
|
||||
ensure!(!busy, "SYSTEM_DELEGATION_UNRESOLVED");
|
||||
let p = intent.permit()?;
|
||||
tx.execute("INSERT INTO system_intents(id,job,attempt,grant_id,body,digest,phase) VALUES(?1,?2,?3,?4,?5,?6,'prepared')",params![intent.id.to_string(),p.job_id.to_string(),p.attempt_id.to_string(),p.grant_id.to_string(),serde_json::to_vec(intent)?,intent.digest()?])?;
|
||||
tx.commit()?;
|
||||
self.check()
|
||||
}
|
||||
pub fn load(&self, id: Uuid) -> Result<(Intent, Phase)> {
|
||||
let (body, digest, phase): (Vec<u8>, String, String) = self.db.query_row(
|
||||
"SELECT body,digest,phase FROM system_intents WHERE id=?1",
|
||||
[id.to_string()],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)?;
|
||||
ensure!(
|
||||
body.len() <= 196608 && hash(&body) == digest,
|
||||
"SYSTEM_INTENT_DIGEST"
|
||||
);
|
||||
let intent: Intent = serde_json::from_slice(&body)?;
|
||||
intent.validate()?;
|
||||
ensure!(intent.id == id, "SYSTEM_INTENT_ID");
|
||||
Ok((intent, Phase::parse(&phase)?))
|
||||
}
|
||||
/// Both cancellation and dispatcher use this same transaction/CAS. A known
|
||||
/// non-dispatch can never follow the conservative may-dispatch marker.
|
||||
pub fn decide(&mut self, id: Uuid, dispatch: bool) -> Result<Phase> {
|
||||
self.check()?;
|
||||
let desired = if dispatch {
|
||||
Phase::MayHaveDispatched
|
||||
} else {
|
||||
Phase::KnownNoDispatch
|
||||
};
|
||||
let (changed, phase) = advance(&mut self.db, id, desired)?;
|
||||
self.check()?;
|
||||
ensure!(
|
||||
changed == 1 || phase != Phase::MayHaveDispatched,
|
||||
"SYSTEM_EXECUTE_REDELIVERY_FORBIDDEN"
|
||||
);
|
||||
Ok(phase)
|
||||
}
|
||||
pub fn cancel_before_dispatch(&mut self, id: Uuid) -> Result<Phase> {
|
||||
self.check()?;
|
||||
let (_, phase) = advance(&mut self.db, id, Phase::KnownNoDispatch)?;
|
||||
self.check()?;
|
||||
Ok(phase)
|
||||
}
|
||||
pub fn pending(&self) -> Result<Vec<Intent>> {
|
||||
let mut rows=self.db.prepare("SELECT i.id FROM system_intents i WHERE NOT EXISTS(SELECT 1 FROM system_intent_mirrors m WHERE m.id=i.id) ORDER BY i.rowid LIMIT 2")?;
|
||||
let ids = rows
|
||||
.query_map([], |r| r.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ensure!(ids.len() <= 1, "SYSTEM_MULTIPLE_PENDING_DELEGATIONS");
|
||||
ids.into_iter()
|
||||
.map(|id| self.load(Uuid::parse_str(&id)?).map(|v| v.0))
|
||||
.collect()
|
||||
}
|
||||
pub fn by_grant(&self, grant: Uuid) -> Result<Intent> {
|
||||
let id: String = self.db.query_row(
|
||||
"SELECT id FROM system_intents WHERE grant_id=?1",
|
||||
[grant.to_string()],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(self.load(Uuid::parse_str(&id)?)?.0)
|
||||
}
|
||||
/// One deny-only attempt per NEW process instance. Original intent/anchor
|
||||
/// bytes and the historical helper process binding are never rewritten.
|
||||
pub fn claim_denial_resume(
|
||||
&mut self,
|
||||
id: Uuid,
|
||||
instance: Uuid,
|
||||
boot: Uuid,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> Result<bool> {
|
||||
self.check()?;
|
||||
let (intent, phase) = self.load(id)?;
|
||||
ensure!(
|
||||
phase == Phase::KnownNoDispatch
|
||||
&& intent.original_boot_id == boot
|
||||
&& intent.agent_instance != instance
|
||||
&& !instance.is_nil()
|
||||
&& observed_at >= intent.recorded_at,
|
||||
"SYSTEM_DENIAL_RESUME_BINDING"
|
||||
);
|
||||
let changed = self.db.execute(
|
||||
CLAIM_RESUME,
|
||||
params![
|
||||
id.to_string(),
|
||||
instance.to_string(),
|
||||
observed_at.to_rfc3339()
|
||||
],
|
||||
)?;
|
||||
self.check()?;
|
||||
Ok(changed == 1)
|
||||
}
|
||||
pub fn mirrored(
|
||||
&mut self,
|
||||
id: Uuid,
|
||||
outcome_sha256: &str,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> Result<()> {
|
||||
self.check()?;
|
||||
ensure!(ipc::digest(outcome_sha256), "SYSTEM_MIRROR_HASH");
|
||||
let old: Option<String> = self
|
||||
.db
|
||||
.query_row(
|
||||
"SELECT outcome_sha256 FROM system_intent_mirrors WHERE id=?1",
|
||||
[id.to_string()],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
if let Some(old) = old {
|
||||
ensure!(old == outcome_sha256, "SYSTEM_MIRROR_CONFLICT");
|
||||
return Ok(());
|
||||
}
|
||||
self.db.execute(
|
||||
"INSERT INTO system_intent_mirrors VALUES(?1,?2,?3)",
|
||||
params![id.to_string(), outcome_sha256, observed_at.to_rfc3339()],
|
||||
)?;
|
||||
self.check()
|
||||
}
|
||||
pub fn verify_no_dispatch(
|
||||
&mut self,
|
||||
id: Uuid,
|
||||
digest: &str,
|
||||
original: &Request,
|
||||
anchor: &NoStartAnchor,
|
||||
) -> Result<NoDispatchObservation> {
|
||||
self.check()?;
|
||||
let (intent, phase) = self.load(id)?;
|
||||
ensure!(
|
||||
phase == Phase::KnownNoDispatch
|
||||
&& intent.digest()? == digest
|
||||
&& serde_json::to_vec(&intent.request)? == serde_json::to_vec(original)?
|
||||
&& serde_json::to_vec(&intent.anchor)? == serde_json::to_vec(anchor)?,
|
||||
"SYSTEM_NO_DISPATCH_PROOF_INVALID"
|
||||
);
|
||||
self.check()?;
|
||||
Ok(NoDispatchObservation {
|
||||
intent_id: id,
|
||||
intent_sha256: digest.into(),
|
||||
intent_recorded_at: intent.recorded_at,
|
||||
original_boot_id: intent.original_boot_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn cancel_and_dispatch_cas_are_exclusive_and_history_cannot_be_rewritten() {
|
||||
let mut db = Connection::open_in_memory().unwrap();
|
||||
db.execute_batch(SCHEMA).unwrap();
|
||||
let cancelled = Uuid::new_v4();
|
||||
let dispatched = Uuid::new_v4();
|
||||
for id in [cancelled, dispatched] {
|
||||
db.execute(
|
||||
"INSERT INTO system_intents VALUES(?1,?1,?1,?1,x'01','immutable','prepared')",
|
||||
[id.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
advance(&mut db, cancelled, Phase::KnownNoDispatch).unwrap(),
|
||||
(1, Phase::KnownNoDispatch)
|
||||
);
|
||||
assert_eq!(
|
||||
advance(&mut db, cancelled, Phase::MayHaveDispatched).unwrap(),
|
||||
(0, Phase::KnownNoDispatch)
|
||||
);
|
||||
assert_eq!(
|
||||
advance(&mut db, dispatched, Phase::MayHaveDispatched).unwrap(),
|
||||
(1, Phase::MayHaveDispatched)
|
||||
);
|
||||
assert_eq!(
|
||||
advance(&mut db, dispatched, Phase::KnownNoDispatch).unwrap(),
|
||||
(0, Phase::MayHaveDispatched)
|
||||
);
|
||||
assert!(
|
||||
db.execute(
|
||||
"UPDATE system_intents SET phase='prepared' WHERE id=?1",
|
||||
[dispatched.to_string()]
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
db.execute(
|
||||
"UPDATE system_intents SET body=x'02' WHERE id=?1",
|
||||
[cancelled.to_string()]
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
db.execute(
|
||||
"DELETE FROM system_intents WHERE id=?1",
|
||||
[cancelled.to_string()]
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let retained: u64 = db
|
||||
.query_row("SELECT count(*) FROM system_intents", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(retained, 2);
|
||||
let resumed = Uuid::new_v4();
|
||||
let args = params![
|
||||
cancelled.to_string(),
|
||||
resumed.to_string(),
|
||||
Utc::now().to_rfc3339()
|
||||
];
|
||||
assert_eq!(db.execute(CLAIM_RESUME, args).unwrap(), 1);
|
||||
assert_eq!(db.execute(CLAIM_RESUME, args).unwrap(), 0);
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use rmm_agent_core::{
|
||||
auth::{Binding, Outcome, Request},
|
||||
helper_reconcile::{SignedResponse, Snapshot},
|
||||
helper_reconcile::{NoStartAnchor, SignedResponse, Snapshot},
|
||||
};
|
||||
use rmm_domain::jobs::hash;
|
||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
@@ -34,6 +34,176 @@ pub const FRAME_LIMIT: usize = 1024 * 1024;
|
||||
pub const CLIENT_PIPE_RIGHTS: u32 = 0x0012_0083; // data R/W, attributes R, RC, synchronize; no bit4
|
||||
const READ_MASK: u32 = 0x0012_00a9;
|
||||
|
||||
/// Static diagnostic contexts only; no caller input or error text is serialized.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum Stage {
|
||||
DiscoverPin,
|
||||
ClientSelfRole,
|
||||
ClientOpenPipe,
|
||||
ClientHelperPeer,
|
||||
ClientSendFrame,
|
||||
ClientReadReply,
|
||||
ClientReplyBinding,
|
||||
ClientReport,
|
||||
ServerConnect,
|
||||
ServerReadHeader,
|
||||
ServerClientPeer,
|
||||
ServerImpersonate,
|
||||
ServerVerifyToken,
|
||||
ServerReadBody,
|
||||
ServerCallBinding,
|
||||
ServerRecheckPeer,
|
||||
ServerReply,
|
||||
Unclassified,
|
||||
}
|
||||
impl Stage {
|
||||
fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::DiscoverPin => "discover_pin",
|
||||
Self::ClientSelfRole => "client_self_role",
|
||||
Self::ClientOpenPipe => "client_open_pipe",
|
||||
Self::ClientHelperPeer => "client_helper_peer",
|
||||
Self::ClientSendFrame => "client_send_frame",
|
||||
Self::ClientReadReply => "client_read_reply",
|
||||
Self::ClientReplyBinding => "client_reply_binding",
|
||||
Self::ClientReport => "client_report",
|
||||
Self::ServerConnect => "server_connect",
|
||||
Self::ServerReadHeader => "server_read_header",
|
||||
Self::ServerClientPeer => "server_client_peer",
|
||||
Self::ServerImpersonate => "server_impersonate",
|
||||
Self::ServerVerifyToken => "server_verify_client_token",
|
||||
Self::ServerReadBody => "server_read_body",
|
||||
Self::ServerCallBinding => "server_call_binding",
|
||||
Self::ServerRecheckPeer => "server_recheck_peer",
|
||||
Self::ServerReply => "server_reply",
|
||||
Self::Unclassified => "unclassified",
|
||||
}
|
||||
}
|
||||
}
|
||||
impl std::fmt::Display for Stage {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.name())
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
pub struct Diagnostic {
|
||||
pub stage: &'static str,
|
||||
pub code: &'static str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub win32: Option<u32>,
|
||||
}
|
||||
impl std::fmt::Display for Diagnostic {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.code)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for Diagnostic {}
|
||||
#[derive(Debug)]
|
||||
struct CapturedWin32(u32);
|
||||
impl std::fmt::Display for CapturedWin32 {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("WIN32_CAPTURED")
|
||||
}
|
||||
}
|
||||
impl std::error::Error for CapturedWin32 {}
|
||||
fn api_failure(code: &'static str) -> anyhow::Error {
|
||||
let captured = unsafe { GetLastError() };
|
||||
anyhow::anyhow!(code).context(CapturedWin32(captured))
|
||||
}
|
||||
fn known_code(text: &str) -> Option<&'static str> {
|
||||
const CODES: &[&str] = &[
|
||||
"HELPER_BINARY_HASH",
|
||||
"HELPER_CALL_BINDING",
|
||||
"HELPER_EMPTY_DACL",
|
||||
"HELPER_EXCESSIVE_WRITER_RIGHTS",
|
||||
"HELPER_FILE_ID",
|
||||
"HELPER_FILE_ID_CHANGED",
|
||||
"HELPER_FILE_INFORMATION",
|
||||
"HELPER_FILE_TYPE",
|
||||
"HELPER_FRAME_CAPACITY",
|
||||
"HELPER_FRAME_EMPTY",
|
||||
"HELPER_FRAME_LENGTH",
|
||||
"HELPER_HANDLE",
|
||||
"HELPER_IMAGE_PARENT",
|
||||
"HELPER_IMAGE_SIZE",
|
||||
"HELPER_IMPERSONATED_SELF",
|
||||
"HELPER_LOCAL_ABSOLUTE_PATH",
|
||||
"HELPER_NETWORK_AGENT_ADMIN_FORBIDDEN",
|
||||
"HELPER_NULL_DACL",
|
||||
"HELPER_PATH_ACE",
|
||||
"HELPER_PATH_ACE_TYPE",
|
||||
"HELPER_PATH_OWNER",
|
||||
"HELPER_PATH_REPLACED",
|
||||
"HELPER_PATH_SECURITY",
|
||||
"HELPER_PATH_WRITABLE_OR_FOREIGN",
|
||||
"HELPER_PEER_RESTARTED",
|
||||
"HELPER_PEER_TOKEN_QUERY",
|
||||
"HELPER_PEER_TOKEN_RECHECK",
|
||||
"HELPER_PIN_PROFILE",
|
||||
"HELPER_PIPE_CONNECT",
|
||||
"HELPER_PIPE_EOF",
|
||||
"HELPER_PIPE_IMPERSONATED_TOKEN",
|
||||
"HELPER_PIPE_IMPERSONATION",
|
||||
"HELPER_PIPE_IO",
|
||||
"HELPER_PIPE_SCM_PID",
|
||||
"HELPER_PIPE_TIMEOUT",
|
||||
"HELPER_PIPE_UNAVAILABLE_OR_BUSY",
|
||||
"HELPER_PROCESS_IDENTITY",
|
||||
"HELPER_PROCESS_IMAGE",
|
||||
"HELPER_PROCESS_IMAGE_BINDING",
|
||||
"HELPER_PROCESS_IMAGE_IDENTITY",
|
||||
"HELPER_PROCESS_TIME",
|
||||
"HELPER_PROCESS_TOKEN_IDENTITY",
|
||||
"HELPER_REPARSE_ANCESTOR",
|
||||
"HELPER_REPLY_BINDING",
|
||||
"HELPER_REPLY_OPERATION",
|
||||
"HELPER_SCM_CONFIG",
|
||||
"HELPER_SCM_CONFIG_SIZE",
|
||||
"HELPER_SCM_IMAGE_BINDING",
|
||||
"HELPER_SCM_MANAGER",
|
||||
"HELPER_SCM_NOT_RUNNING",
|
||||
"HELPER_SCM_SERVICE",
|
||||
"HELPER_SELF_TOKEN",
|
||||
"HELPER_SERVICE_SID_NOT_ENABLED",
|
||||
"HELPER_THREAD_TOKEN_UNVERIFIED",
|
||||
"HELPER_TOKEN_GROUPS_SIZE",
|
||||
"HELPER_TOKEN_INFORMATION",
|
||||
"HELPER_TOKEN_SIZE",
|
||||
"HELPER_TOKEN_USER",
|
||||
"SYSTEM_LEDGER_OR_LOCK_REPLACED",
|
||||
"SYSTEM_HELD_FILE_CHANGED",
|
||||
"SYSTEM_DB_FILE_ID",
|
||||
"SYSTEM_CALL_BINDING",
|
||||
];
|
||||
CODES.iter().copied().find(|code| *code == text)
|
||||
}
|
||||
pub fn diagnostic(error: &anyhow::Error) -> Diagnostic {
|
||||
if let Some(value) = error.downcast_ref::<Diagnostic>() {
|
||||
return *value;
|
||||
}
|
||||
let code = error
|
||||
.chain()
|
||||
.find_map(|cause| known_code(&cause.to_string()))
|
||||
.unwrap_or("UNCLASSIFIED_IO_OR_CODEC");
|
||||
Diagnostic {
|
||||
stage: error
|
||||
.downcast_ref::<Stage>()
|
||||
.copied()
|
||||
.unwrap_or(Stage::Unclassified)
|
||||
.name(),
|
||||
code,
|
||||
win32: error.downcast_ref::<CapturedWin32>().map(|v| v.0),
|
||||
}
|
||||
}
|
||||
pub fn remote_denial(stage: Stage, code: &str) -> anyhow::Error {
|
||||
Diagnostic {
|
||||
stage: stage.name(),
|
||||
code: known_code(code).unwrap_or("UNCLASSIFIED_REMOTE_DENIAL"),
|
||||
win32: None,
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PeerPin {
|
||||
@@ -118,6 +288,12 @@ pub enum Operation {
|
||||
generation: u64,
|
||||
request: Request,
|
||||
},
|
||||
Abandon {
|
||||
anchor: Box<NoStartAnchor>,
|
||||
request: Request,
|
||||
intent_id: Uuid,
|
||||
intent_sha256: String,
|
||||
},
|
||||
}
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -155,10 +331,14 @@ pub enum ReplyBody {
|
||||
snapshot: Snapshot,
|
||||
unlocked_grants: Vec<Uuid>,
|
||||
ready: bool,
|
||||
no_start_anchor: Option<Box<NoStartAnchor>>,
|
||||
},
|
||||
Executed {
|
||||
outcome: Outcome,
|
||||
},
|
||||
Abandoned {
|
||||
outcome: Outcome,
|
||||
},
|
||||
Denied {
|
||||
code: String,
|
||||
},
|
||||
@@ -191,10 +371,9 @@ pub fn wide(s: impl AsRef<std::ffi::OsStr>) -> Vec<u16> {
|
||||
s.as_ref().encode_wide().chain(Some(0)).collect()
|
||||
}
|
||||
fn owned(raw: HANDLE) -> Result<OwnedHandle> {
|
||||
ensure!(
|
||||
!raw.is_null() && raw != INVALID_HANDLE_VALUE,
|
||||
"HELPER_HANDLE"
|
||||
);
|
||||
if raw.is_null() || raw == INVALID_HANDLE_VALUE {
|
||||
return Err(api_failure("HELPER_HANDLE"));
|
||||
}
|
||||
Ok(unsafe { OwnedHandle::from_raw_handle(raw) })
|
||||
}
|
||||
fn sid_text(sid: PSID) -> Result<String> {
|
||||
@@ -323,7 +502,12 @@ pub fn require_self(peer: &PeerPin) -> Result<()> {
|
||||
let token = owned(token)?;
|
||||
verify_token(token.as_raw_handle(), &peer.user_sid, &peer.service_sid)
|
||||
}
|
||||
fn check_acl(handle: HANDLE, readers: &[String], writers: &[String]) -> Result<()> {
|
||||
fn check_acl(
|
||||
handle: HANDLE,
|
||||
readers: &[String],
|
||||
writers: &[String],
|
||||
writer_mask: Option<u32>,
|
||||
) -> Result<()> {
|
||||
let mut owner = ptr::null_mut();
|
||||
let mut acl = ptr::null_mut();
|
||||
let mut descriptor = ptr::null_mut();
|
||||
@@ -365,6 +549,12 @@ fn check_acl(handle: HANDLE, readers: &[String], writers: &[String]) -> Result<(
|
||||
let sid = sid_text(ptr::addr_of!(ace.SidStart).cast_mut().cast())?;
|
||||
let privileged =
|
||||
matches!(sid.as_str(), "S-1-5-18" | "S-1-5-32-544") || writers.contains(&sid);
|
||||
if writers.contains(&sid) && !matches!(sid.as_str(), "S-1-5-18" | "S-1-5-32-544") {
|
||||
ensure!(
|
||||
writer_mask.is_none_or(|mask| ace.Mask & !mask == 0),
|
||||
"HELPER_EXCESSIVE_WRITER_RIGHTS"
|
||||
);
|
||||
}
|
||||
ensure!(
|
||||
privileged || (readers.contains(&sid) && ace.Mask & !READ_MASK == 0),
|
||||
"HELPER_PATH_WRITABLE_OR_FOREIGN"
|
||||
@@ -384,6 +574,7 @@ pub struct PinnedFile {
|
||||
readers: Vec<String>,
|
||||
writers: Vec<String>,
|
||||
directory: bool,
|
||||
writer_mask: Option<u32>,
|
||||
}
|
||||
impl PinnedFile {
|
||||
pub fn open(
|
||||
@@ -392,6 +583,28 @@ impl PinnedFile {
|
||||
readers: &[String],
|
||||
writers: &[String],
|
||||
write: bool,
|
||||
) -> Result<Self> {
|
||||
Self::open_impl(path, directory, readers, writers, write, false)
|
||||
}
|
||||
pub(crate) fn open_read_shared(
|
||||
path: &Path,
|
||||
directory: bool,
|
||||
readers: &[String],
|
||||
writers: &[String],
|
||||
) -> Result<Self> {
|
||||
Self::open_impl(path, directory, readers, writers, false, true)
|
||||
}
|
||||
pub(crate) fn limit_writer_rights(&mut self, mask: u32) -> Result<()> {
|
||||
self.writer_mask = Some(mask);
|
||||
self.check()
|
||||
}
|
||||
fn open_impl(
|
||||
path: &Path,
|
||||
directory: bool,
|
||||
readers: &[String],
|
||||
writers: &[String],
|
||||
write: bool,
|
||||
share_write: bool,
|
||||
) -> Result<Self> {
|
||||
ensure!(
|
||||
path.is_absolute()
|
||||
@@ -427,7 +640,12 @@ impl PinnedFile {
|
||||
} else {
|
||||
GENERIC_READ
|
||||
},
|
||||
FILE_SHARE_READ | if write { FILE_SHARE_WRITE } else { 0 },
|
||||
FILE_SHARE_READ
|
||||
| if write || share_write {
|
||||
FILE_SHARE_WRITE
|
||||
} else {
|
||||
0
|
||||
},
|
||||
ptr::null(),
|
||||
OPEN_EXISTING,
|
||||
FILE_FLAG_OPEN_REPARSE_POINT
|
||||
@@ -448,6 +666,7 @@ impl PinnedFile {
|
||||
readers: readers.into(),
|
||||
writers: writers.into(),
|
||||
directory,
|
||||
writer_mask: None,
|
||||
};
|
||||
value.check()?;
|
||||
Ok(value)
|
||||
@@ -467,7 +686,12 @@ impl PinnedFile {
|
||||
file_identity(&self.file)? == self.identity,
|
||||
"HELPER_FILE_ID_CHANGED"
|
||||
);
|
||||
check_acl(self.file.as_raw_handle(), &self.readers, &self.writers)?;
|
||||
check_acl(
|
||||
self.file.as_raw_handle(),
|
||||
&self.readers,
|
||||
&self.writers,
|
||||
self.writer_mask,
|
||||
)?;
|
||||
let raw = unsafe {
|
||||
CreateFileW(
|
||||
wide(&self.path).as_ptr(),
|
||||
@@ -877,10 +1101,30 @@ impl IoEvent {
|
||||
}
|
||||
let mut count = 0;
|
||||
let complete = unsafe { GetOverlappedResult(pipe, &self.overlapped, &mut count, 0) };
|
||||
if complete == 0 && unsafe { GetLastError() } == ERROR_IO_INCOMPLETE {
|
||||
std::process::abort();
|
||||
if complete == 0 {
|
||||
let error = api_failure("HELPER_PIPE_IO");
|
||||
if error
|
||||
.downcast_ref::<CapturedWin32>()
|
||||
.is_some_and(|v| v.0 == ERROR_IO_INCOMPLETE)
|
||||
{
|
||||
std::process::abort();
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
fn completed(&self, pipe: HANDLE) -> Result<u32> {
|
||||
let mut count = 0;
|
||||
if unsafe { GetOverlappedResult(pipe, &self.overlapped, &mut count, 0) } == 0 {
|
||||
let error = api_failure("HELPER_PIPE_IO");
|
||||
if error
|
||||
.downcast_ref::<CapturedWin32>()
|
||||
.is_some_and(|v| v.0 == ERROR_IO_INCOMPLETE)
|
||||
{
|
||||
std::process::abort();
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
ensure!(complete != 0, "HELPER_PIPE_IO");
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
@@ -890,14 +1134,13 @@ fn transfer(pipe: HANDLE, bytes: &mut [u8], write: bool, deadline: Instant) -> R
|
||||
ensure!(Instant::now() < deadline, "HELPER_PIPE_TIMEOUT");
|
||||
let mut operation = IoEvent::new()?;
|
||||
let part = &mut bytes[offset..];
|
||||
let mut done = 0;
|
||||
let ok = unsafe {
|
||||
if write {
|
||||
WriteFile(
|
||||
pipe,
|
||||
part.as_ptr(),
|
||||
part.len().try_into()?,
|
||||
&mut done,
|
||||
ptr::null_mut(),
|
||||
&mut operation.overlapped,
|
||||
)
|
||||
} else {
|
||||
@@ -905,18 +1148,21 @@ fn transfer(pipe: HANDLE, bytes: &mut [u8], write: bool, deadline: Instant) -> R
|
||||
pipe,
|
||||
part.as_mut_ptr(),
|
||||
part.len().try_into()?,
|
||||
&mut done,
|
||||
ptr::null_mut(),
|
||||
&mut operation.overlapped,
|
||||
)
|
||||
}
|
||||
};
|
||||
let count = if ok != 0 {
|
||||
done
|
||||
// ReadFile/WriteFile are always OVERLAPPED here. Their direct
|
||||
// byte-counter is unspecified for async handles, even on TRUE.
|
||||
// Obtain completion bytes from this exact held OVERLAPPED instead.
|
||||
operation.completed(pipe)?
|
||||
} else {
|
||||
ensure!(
|
||||
unsafe { GetLastError() } == ERROR_IO_PENDING,
|
||||
"HELPER_PIPE_IO"
|
||||
);
|
||||
let error = unsafe { GetLastError() };
|
||||
if error != ERROR_IO_PENDING {
|
||||
return Err(anyhow::anyhow!("HELPER_PIPE_IO").context(CapturedWin32(error)));
|
||||
}
|
||||
operation.wait(pipe, deadline)?
|
||||
};
|
||||
ensure!(count > 0 && count as usize <= part.len(), "HELPER_PIPE_EOF");
|
||||
@@ -1000,37 +1246,42 @@ impl Server {
|
||||
}
|
||||
pub fn receive(&self) -> Result<Option<Received>> {
|
||||
let pipe = self.pipe.as_raw_handle();
|
||||
let mut connect = IoEvent::new()?;
|
||||
let mut connect = IoEvent::new().context(Stage::ServerConnect)?;
|
||||
let connected = unsafe { ConnectNamedPipe(pipe, &mut connect.overlapped) };
|
||||
if connected == 0 {
|
||||
let error = unsafe { GetLastError() };
|
||||
if error == ERROR_IO_PENDING {
|
||||
if connect
|
||||
.wait(pipe, Instant::now() + Duration::from_millis(250))
|
||||
.is_err()
|
||||
if let Err(error) = connect.wait(pipe, Instant::now() + Duration::from_millis(250))
|
||||
{
|
||||
unsafe {
|
||||
DisconnectNamedPipe(pipe);
|
||||
if diagnostic(&error).code == "HELPER_PIPE_TIMEOUT" {
|
||||
unsafe {
|
||||
DisconnectNamedPipe(pipe);
|
||||
};
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(None);
|
||||
return Err(error).context(Stage::ServerConnect);
|
||||
}
|
||||
} else {
|
||||
ensure!(error == ERROR_PIPE_CONNECTED, "HELPER_PIPE_CONNECT");
|
||||
if error != ERROR_PIPE_CONNECTED {
|
||||
return Err(
|
||||
anyhow::anyhow!("HELPER_PIPE_CONNECT").context(CapturedWin32(error))
|
||||
)
|
||||
.context(Stage::ServerConnect);
|
||||
}
|
||||
}
|
||||
}
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let len = length(pipe, deadline)?;
|
||||
let mut peer = PeerProof::capture(pipe, &self.pin.agent, true)?;
|
||||
ensure!(
|
||||
unsafe { ImpersonateNamedPipeClient(pipe) } != 0,
|
||||
"HELPER_PIPE_IMPERSONATION"
|
||||
);
|
||||
let len = length(pipe, deadline).context(Stage::ServerReadHeader)?;
|
||||
let mut peer =
|
||||
PeerProof::capture(pipe, &self.pin.agent, true).context(Stage::ServerClientPeer)?;
|
||||
if unsafe { ImpersonateNamedPipeClient(pipe) } == 0 {
|
||||
return Err(api_failure("HELPER_PIPE_IMPERSONATION")).context(Stage::ServerImpersonate);
|
||||
}
|
||||
let auth = (|| {
|
||||
let mut token = ptr::null_mut();
|
||||
ensure!(
|
||||
unsafe { OpenThreadToken(GetCurrentThread(), TOKEN_QUERY, 1, &mut token) } != 0,
|
||||
"HELPER_PIPE_IMPERSONATED_TOKEN"
|
||||
);
|
||||
if unsafe { OpenThreadToken(GetCurrentThread(), TOKEN_QUERY, 1, &mut token) } == 0 {
|
||||
return Err(api_failure("HELPER_PIPE_IMPERSONATED_TOKEN"));
|
||||
}
|
||||
let token = owned(token)?;
|
||||
verify_token(
|
||||
token.as_raw_handle(),
|
||||
@@ -1041,17 +1292,17 @@ impl Server {
|
||||
if unsafe { RevertToSelf() } == 0 {
|
||||
std::process::abort();
|
||||
}
|
||||
auth?;
|
||||
require_self(&self.pin.helper)?;
|
||||
let call: Call = body(pipe, len, deadline)?;
|
||||
ensure!(
|
||||
call.schema_version == 1
|
||||
&& !call.challenge.is_nil()
|
||||
&& call.policy_sha256 == self.pin.policy_sha256
|
||||
&& binding_hash(&call.binding)? == binding_hash(&self.pin.binding)?,
|
||||
"HELPER_CALL_BINDING"
|
||||
);
|
||||
peer.check()?;
|
||||
auth.context(Stage::ServerVerifyToken)?;
|
||||
require_self(&self.pin.helper).context(Stage::ServerVerifyToken)?;
|
||||
let call: Call = body(pipe, len, deadline).context(Stage::ServerReadBody)?;
|
||||
if !(call.schema_version == 1
|
||||
&& !call.challenge.is_nil()
|
||||
&& call.policy_sha256 == self.pin.policy_sha256
|
||||
&& binding_hash(&call.binding)? == binding_hash(&self.pin.binding)?)
|
||||
{
|
||||
return Err(anyhow::anyhow!("HELPER_CALL_BINDING")).context(Stage::ServerCallBinding);
|
||||
}
|
||||
peer.check().context(Stage::ServerRecheckPeer)?;
|
||||
Ok(Some(Received { call, peer }))
|
||||
}
|
||||
pub fn reply(&self, value: &Reply) -> Result<()> {
|
||||
@@ -1060,6 +1311,7 @@ impl Server {
|
||||
value,
|
||||
Instant::now() + Duration::from_secs(5),
|
||||
)
|
||||
.context(Stage::ServerReply)
|
||||
}
|
||||
pub fn disconnect(&self) {
|
||||
unsafe {
|
||||
@@ -1070,7 +1322,7 @@ impl Server {
|
||||
/// Blocking SDK call; run on the agent's bounded blocking worker, never heartbeat reactor.
|
||||
pub fn exchange(pin: &Pin, call: &Call, timeout: Duration) -> Result<Reply> {
|
||||
pin.validate()?;
|
||||
require_self(&pin.agent)?;
|
||||
require_self(&pin.agent).context(Stage::ClientSelfRole)?;
|
||||
ensure!(
|
||||
(Duration::from_secs(1)..=Duration::from_secs(125)).contains(&timeout),
|
||||
"HELPER_CALL_TIMEOUT_BOUND"
|
||||
@@ -1087,31 +1339,33 @@ pub fn exchange(pin: &Pin, call: &Call, timeout: Duration) -> Result<Reply> {
|
||||
ptr::null_mut(),
|
||||
)
|
||||
})
|
||||
.context("HELPER_PIPE_UNAVAILABLE_OR_BUSY")?;
|
||||
let mut peer = PeerProof::capture(pipe.as_raw_handle(), &pin.helper, false)?;
|
||||
send(pipe.as_raw_handle(), call, deadline)?;
|
||||
let len = length(pipe.as_raw_handle(), deadline)?;
|
||||
let reply: Reply = body(pipe.as_raw_handle(), len, deadline)?;
|
||||
peer.check()?;
|
||||
ensure!(
|
||||
reply.schema_version == 1
|
||||
&& reply.challenge == call.challenge
|
||||
&& reply.binding_sha256 == binding_hash(&pin.binding)?
|
||||
&& reply.policy_sha256 == pin.policy_sha256
|
||||
&& reply.helper_binary_sha256 == pin.helper.binary_sha256,
|
||||
"HELPER_REPLY_BINDING"
|
||||
);
|
||||
ensure!(
|
||||
matches!(
|
||||
(&call.operation, &reply.body),
|
||||
(_, ReplyBody::Denied { .. })
|
||||
| (Operation::Probe { .. }, ReplyBody::Probe { .. })
|
||||
| (Operation::Snapshot { .. }, ReplyBody::Snapshot { .. })
|
||||
| (Operation::Apply { .. }, ReplyBody::Applied { .. })
|
||||
| (Operation::Execute { .. }, ReplyBody::Executed { .. })
|
||||
),
|
||||
"HELPER_REPLY_OPERATION"
|
||||
);
|
||||
.context("HELPER_PIPE_UNAVAILABLE_OR_BUSY")
|
||||
.context(Stage::ClientOpenPipe)?;
|
||||
let mut peer = PeerProof::capture(pipe.as_raw_handle(), &pin.helper, false)
|
||||
.context(Stage::ClientHelperPeer)?;
|
||||
send(pipe.as_raw_handle(), call, deadline).context(Stage::ClientSendFrame)?;
|
||||
let len = length(pipe.as_raw_handle(), deadline).context(Stage::ClientReadReply)?;
|
||||
let reply: Reply = body(pipe.as_raw_handle(), len, deadline).context(Stage::ClientReadReply)?;
|
||||
peer.check().context(Stage::ClientHelperPeer)?;
|
||||
if !(reply.schema_version == 1
|
||||
&& reply.challenge == call.challenge
|
||||
&& reply.binding_sha256 == binding_hash(&pin.binding)?
|
||||
&& reply.policy_sha256 == pin.policy_sha256
|
||||
&& reply.helper_binary_sha256 == pin.helper.binary_sha256)
|
||||
{
|
||||
return Err(anyhow::anyhow!("HELPER_REPLY_BINDING")).context(Stage::ClientReplyBinding);
|
||||
}
|
||||
if !matches!(
|
||||
(&call.operation, &reply.body),
|
||||
(_, ReplyBody::Denied { .. })
|
||||
| (Operation::Probe { .. }, ReplyBody::Probe { .. })
|
||||
| (Operation::Snapshot { .. }, ReplyBody::Snapshot { .. })
|
||||
| (Operation::Apply { .. }, ReplyBody::Applied { .. })
|
||||
| (Operation::Execute { .. }, ReplyBody::Executed { .. })
|
||||
| (Operation::Abandon { .. }, ReplyBody::Abandoned { .. })
|
||||
) {
|
||||
return Err(anyhow::anyhow!("HELPER_REPLY_OPERATION")).context(Stage::ClientReplyBinding);
|
||||
}
|
||||
Ok(reply)
|
||||
}
|
||||
|
||||
@@ -1155,6 +1409,14 @@ mod tests {
|
||||
assert!(
|
||||
serde_json::from_str::<Operation>(r#"{"operation":"probe","ready":true}"#).is_err()
|
||||
);
|
||||
let caller =
|
||||
anyhow::anyhow!("caller-payload-private-marker").context(Stage::ServerReadBody);
|
||||
let encoded = serde_json::to_string(&diagnostic(&caller)).unwrap();
|
||||
assert!(!encoded.contains("caller-payload-private-marker"));
|
||||
assert_eq!(diagnostic(&caller).stage, "server_read_body");
|
||||
assert_eq!(diagnostic(&caller).code, "UNCLASSIFIED_IO_OR_CODEC");
|
||||
let remote = remote_denial(Stage::ClientReplyBinding, "caller-payload-private-marker");
|
||||
assert_eq!(diagnostic(&remote).code, "UNCLASSIFIED_REMOTE_DENIAL");
|
||||
}
|
||||
#[test]
|
||||
fn reconfigured_service_cannot_adopt_an_old_running_identity() {
|
||||
|
||||
@@ -2,7 +2,12 @@
|
||||
//! advertised here. Each actual IPC runs on one bounded blocking lane.
|
||||
use crate::{helper_ipc as ipc, native};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use rmm_agent_core::{auth::Binding, helper_reconcile::Snapshot, reconcile::HelperSnapshot};
|
||||
use rmm_agent_core::{
|
||||
auth::{Binding, Outcome, Request},
|
||||
helper_reconcile::{NoStartAnchor, Snapshot},
|
||||
reconcile::HelperSnapshot,
|
||||
};
|
||||
use rmm_domain::jobs::{Action, JobPayload};
|
||||
use rmm_protocol::{SyncAttempt, SyncResponse};
|
||||
use serde_json::json;
|
||||
use std::{
|
||||
@@ -12,6 +17,8 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::sync::Semaphore;
|
||||
#[path = "helper_intent.rs"]
|
||||
pub mod intent;
|
||||
pub const PIN_PATH: &str = r"C:\ProgramData\RustDiscoverySystem\agent-pin.json";
|
||||
const PUBLIC_ROOT: &str = r"C:\ProgramData\RustDiscoverySystem";
|
||||
const PROBE_FILE: &str = "helper-probe.json";
|
||||
@@ -20,6 +27,16 @@ pub struct Client {
|
||||
pin: ipc::Pin,
|
||||
lane: Arc<Semaphore>,
|
||||
}
|
||||
pub struct ControlSnapshot {
|
||||
pub raw: Snapshot,
|
||||
pub no_start_anchor: Option<NoStartAnchor>,
|
||||
}
|
||||
pub struct BoundOutcome {
|
||||
pub job_id: uuid::Uuid,
|
||||
pub attempt_id: uuid::Uuid,
|
||||
pub grant_id: uuid::Uuid,
|
||||
pub outcome: Outcome,
|
||||
}
|
||||
impl Client {
|
||||
/// Missing pin is deliberately passive. A present but invalid pin is refused;
|
||||
/// no endpoint/key/policy is discovered from the pipe itself.
|
||||
@@ -72,12 +89,13 @@ impl Client {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _permit = permit;
|
||||
let result=(|| {
|
||||
let client=Self::discover(&agent_id,&asset_id,binding.as_ref(),lane)?;
|
||||
let client=Self::discover(&agent_id,&asset_id,binding.as_ref(),lane).context(ipc::Stage::DiscoverPin)?;
|
||||
if let Some(client)=&client {client.probe_blocking(&state)?;}
|
||||
Ok(client)
|
||||
})();
|
||||
if result.is_err() || matches!(&result,Ok(None)) && state.join(PROBE_FILE).try_exists()? {
|
||||
write_current_probe(&state,&json!({"schema_version":1,"source":"actual_local_service_runtime","agent_process_id":std::process::id(),"agent_id":agent_id,"asset_id":asset_id,"observed_at":chrono::Utc::now(),"authenticated_pipe_exchange":false,"system_capability_advertised":false,"helper_ready":false,"no_jobs":true,"code":"HELPER_NOT_AVAILABLE"}))?;
|
||||
let diagnostic=result.as_ref().err().map(ipc::diagnostic).unwrap_or(ipc::Diagnostic{stage:"discover_pin",code:"HELPER_PIN_NOT_CONFIGURED",win32:None});
|
||||
write_current_probe(&state,&json!({"schema_version":1,"source":"actual_local_service_runtime","agent_process_id":std::process::id(),"agent_id":agent_id,"asset_id":asset_id,"observed_at":chrono::Utc::now(),"authenticated_pipe_exchange":false,"system_capability_advertised":false,"helper_ready":false,"no_jobs":true,"code":"HELPER_NOT_AVAILABLE","diagnostic":diagnostic}))?;
|
||||
}
|
||||
result
|
||||
})
|
||||
@@ -100,7 +118,135 @@ impl Client {
|
||||
pub fn pin_hash(&self) -> &str {
|
||||
&self.pin.policy_sha256
|
||||
}
|
||||
pub fn helper_binary_hash(&self) -> &str {
|
||||
&self.pin.helper.binary_sha256
|
||||
}
|
||||
/// Import only independently authenticated raw helper evidence. c+start is
|
||||
/// atomic through the existing core API. A crash before outcome persistence
|
||||
/// stays locked; the next fresh snapshot repairs that exact start idempotently.
|
||||
pub fn mirror(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
ledger: &mut rusqlite::Connection,
|
||||
observed_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> Result<Vec<BoundOutcome>> {
|
||||
use rmm_agent_core::{crypto::DalekVerifier, ledger as core_ledger};
|
||||
use rusqlite::OptionalExtension;
|
||||
if !Path::new(intent::DB).try_exists()? {
|
||||
// Optional provisioning must not block the unprivileged provider.
|
||||
// Retained SYSTEM consumption, however, may never lose its store.
|
||||
let mut rows = ledger.prepare("SELECT start FROM execution_evidence")?;
|
||||
for row in rows.query_map([], |r| r.get::<_, Vec<u8>>(0))? {
|
||||
let value: serde_json::Value = serde_json::from_slice(&row?)?;
|
||||
let request: Request = serde_json::from_value(value["request"].clone())?;
|
||||
ensure!(
|
||||
self.pin.binding.job(&request.job_bytes)?.execution_identity != "system",
|
||||
"SYSTEM_INTENT_STORE_MISSING"
|
||||
);
|
||||
}
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut store = intent::Store::open(true)?;
|
||||
for intent in store.pending()? {
|
||||
self.pin.binding.verify_historical_for_denial_with(
|
||||
&intent.request,
|
||||
rmm_agent_core::auth::HelperIdentity::System,
|
||||
&DalekVerifier,
|
||||
)?;
|
||||
let job = self.pin.binding.job(&intent.request.job_bytes)?;
|
||||
let permit = intent.permit()?;
|
||||
ensure!(
|
||||
self.supports_system(&job)
|
||||
&& intent.policy_sha256 == self.pin.policy_sha256
|
||||
&& intent.helper_binary_sha256 == self.pin.helper.binary_sha256
|
||||
&& snapshot.boot_id == intent.original_boot_id
|
||||
&& snapshot.journal_id == intent.anchor.journal_id()
|
||||
&& snapshot.checkpoint >= intent.anchor.checkpoint()
|
||||
&& observed_at >= intent.recorded_at
|
||||
&& observed_at >= snapshot.captured_at,
|
||||
"SYSTEM_MIRROR_CONTEXT"
|
||||
);
|
||||
let mut matches = snapshot
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|a| a.job_id == job.job_id && a.attempt_id == job.attempt_id);
|
||||
let row = matches
|
||||
.next()
|
||||
.context("SYSTEM_DELEGATION_NO_HELPER_EVIDENCE")?;
|
||||
ensure!(matches.next().is_none(), "SYSTEM_DUPLICATE_HELPER_ATTEMPT");
|
||||
let outcome =
|
||||
verified_mirror_row(row, &job, &permit, &intent.request, intent.original_boot_id)?;
|
||||
let existing:Option<(String,u64,String,String,Vec<u8>)>=ledger.query_row("SELECT c.agent,c.generation,c.job,c.attempt,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?1",[permit.grant_id.to_string()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?))).optional()?;
|
||||
if let Some((agent, generation, j, a, start)) = existing {
|
||||
let start: serde_json::Value = serde_json::from_slice(&start)?;
|
||||
let request: Request = serde_json::from_value(start["request"].clone())?;
|
||||
ensure!(
|
||||
agent == job.agent_id.to_string()
|
||||
&& generation == job.enrollment_generation
|
||||
&& j == job.job_id.to_string()
|
||||
&& a == job.attempt_id.to_string()
|
||||
&& serde_json::to_vec(&request)? == serde_json::to_vec(&intent.request)?
|
||||
&& start["previous_boot_id"] == intent.original_boot_id.to_string()
|
||||
&& serde_json::from_value::<chrono::DateTime<chrono::Utc>>(
|
||||
start["reserved_at"].clone()
|
||||
)? == intent.recorded_at,
|
||||
"SYSTEM_MIRROR_LOCAL_START_CONFLICT"
|
||||
);
|
||||
} else {
|
||||
core_ledger::begin_execution(
|
||||
ledger,
|
||||
&permit,
|
||||
&intent.request,
|
||||
intent.original_boot_id,
|
||||
intent.recorded_at,
|
||||
)?;
|
||||
}
|
||||
core_ledger::persist_outcome(ledger, &permit, &outcome)?;
|
||||
store.mirrored(intent.id, &row.outcome_sha256, observed_at)?;
|
||||
}
|
||||
let mut unknowns = Vec::new();
|
||||
for row in snapshot
|
||||
.attempts
|
||||
.iter()
|
||||
.filter(|row| row.state == "unknown")
|
||||
{
|
||||
let intent = store.by_grant(row.grant_id)?;
|
||||
self.pin.binding.verify_historical_for_denial_with(
|
||||
&intent.request,
|
||||
rmm_agent_core::auth::HelperIdentity::System,
|
||||
&DalekVerifier,
|
||||
)?;
|
||||
let job = self.pin.binding.job(&intent.request.job_bytes)?;
|
||||
let permit = intent.permit()?;
|
||||
ensure!(
|
||||
self.supports_system(&job)
|
||||
&& intent.policy_sha256 == self.pin.policy_sha256
|
||||
&& intent.helper_binary_sha256 == self.pin.helper.binary_sha256
|
||||
&& snapshot.boot_id == intent.original_boot_id
|
||||
&& snapshot.journal_id == intent.anchor.journal_id()
|
||||
&& snapshot.checkpoint >= intent.anchor.checkpoint(),
|
||||
"SYSTEM_MIRROR_CONTEXT"
|
||||
);
|
||||
let outcome =
|
||||
verified_mirror_row(row, &job, &permit, &intent.request, intent.original_boot_id)?;
|
||||
unknowns.push(BoundOutcome {
|
||||
job_id: job.job_id,
|
||||
attempt_id: job.attempt_id,
|
||||
grant_id: permit.grant_id,
|
||||
outcome,
|
||||
});
|
||||
}
|
||||
store.check()?;
|
||||
Ok(unknowns)
|
||||
}
|
||||
async fn call(&self, operation: ipc::Operation) -> Result<ipc::Reply> {
|
||||
self.call_bounded(operation, Duration::from_secs(5)).await
|
||||
}
|
||||
async fn call_bounded(
|
||||
&self,
|
||||
operation: ipc::Operation,
|
||||
timeout: Duration,
|
||||
) -> Result<ipc::Reply> {
|
||||
let permit = self
|
||||
.lane
|
||||
.clone()
|
||||
@@ -111,17 +257,20 @@ impl Client {
|
||||
let _permit = permit;
|
||||
client.check_pin()?;
|
||||
let call = ipc::Call::new(&client.pin, operation);
|
||||
let reply = ipc::exchange(&client.pin, &call, Duration::from_secs(5))?;
|
||||
let reply = ipc::exchange(&client.pin, &call, timeout)?;
|
||||
client.check_pin()?;
|
||||
Ok(reply)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
fn probe_blocking(&self, state: &Path) -> Result<()> {
|
||||
self.check_pin()?;
|
||||
self.check_pin().context(ipc::Stage::DiscoverPin)?;
|
||||
let pin = &self.pin;
|
||||
let call = ipc::Call::new(pin, ipc::Operation::Probe {});
|
||||
let reply = ipc::exchange(pin, &call, Duration::from_secs(5))?;
|
||||
if let ipc::ReplyBody::Denied { code } = &reply.body {
|
||||
return Err(ipc::remote_denial(ipc::Stage::ClientReplyBinding, code));
|
||||
}
|
||||
let ipc::ReplyBody::Probe {
|
||||
clock_qualified,
|
||||
ready,
|
||||
@@ -133,22 +282,123 @@ impl Client {
|
||||
};
|
||||
let value = json!({"schema_version":1,"source":"actual_local_service_runtime","agent_process_id":std::process::id(),"agent_service":crate::SERVICE_NAME,"agent_id":pin.binding.agent_id,"asset_id":pin.binding.asset_id,"challenge":reply.challenge,"observed_at":chrono::Utc::now(),"helper_service":ipc::HELPER_SERVICE,"verified_helper_user_sid":pin.helper.user_sid,"verified_helper_service_sid":pin.helper.service_sid,"helper_binary_sha256":reply.helper_binary_sha256,"policy_sha256":reply.policy_sha256,"authenticated_pipe_exchange":true,"client_service_pid_verified_by_helper":true,"ledger_file_id":ledger,"executor_lock_file_id":lock,"helper_clock_qualified":clock_qualified,"helper_ready":ready,"system_capability_advertised":false,"no_jobs":true});
|
||||
self.check_pin()?;
|
||||
write_current_probe(state, &value)
|
||||
write_current_probe(state, &value).context(ipc::Stage::ClientReport)
|
||||
}
|
||||
pub async fn snapshot(&self, generation: u64, boot: uuid::Uuid) -> Result<HelperSnapshot> {
|
||||
/// Pure admission shape only; no capability or qualification is asserted.
|
||||
pub fn supports_system(&self, job: &JobPayload) -> bool {
|
||||
system_shape(job)
|
||||
&& self.pin.binding.tenant_id == job.tenant_id
|
||||
&& self.pin.binding.asset_id == job.asset_id
|
||||
&& self.pin.binding.agent_id == job.agent_id
|
||||
&& self.pin.binding.enrollment_generation == job.enrollment_generation
|
||||
&& self.pin.binding.assignment_version == job.assignment_version
|
||||
}
|
||||
/// Delegate ONCE. A lost reply is unknown and must be recovered by signed
|
||||
/// Snapshot/Apply, never by resending Execute. A timed-out blocking call
|
||||
/// retains its one lane permit until it actually returns; no replacement.
|
||||
pub async fn delegate(
|
||||
&self,
|
||||
intent: intent::Intent,
|
||||
admit: impl Fn() -> Result<()> + Send + 'static,
|
||||
) -> Result<()> {
|
||||
let job = self.pin.binding.job(&intent.request.job_bytes)?;
|
||||
ensure!(
|
||||
intent.connection_generation > 0 && self.supports_system(&job),
|
||||
"SYSTEM_TYPED_SERVICE_REQUIRED"
|
||||
);
|
||||
let timeout = execution_timeout(job.max_runtime_seconds)?;
|
||||
let permit = self
|
||||
.lane
|
||||
.clone()
|
||||
.try_acquire_owned()
|
||||
.context("HELPER_CONTROL_BUSY")?;
|
||||
let client = self.clone();
|
||||
tokio::time::timeout(
|
||||
timeout,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _permit = permit;
|
||||
client.check_pin()?;
|
||||
let mut store = intent::Store::open(true)?;
|
||||
let (retained, _) = store.load(intent.id)?;
|
||||
ensure!(
|
||||
retained.digest()? == intent.digest()?,
|
||||
"SYSTEM_INTENT_CHANGED"
|
||||
);
|
||||
let phase = store.decide(intent.id, admit().is_ok())?;
|
||||
let operation = match phase {
|
||||
intent::Phase::KnownNoDispatch => ipc::Operation::Abandon {
|
||||
anchor: Box::new(intent.anchor.clone()),
|
||||
request: intent.request.clone(),
|
||||
intent_id: intent.id,
|
||||
intent_sha256: intent.digest()?,
|
||||
},
|
||||
intent::Phase::MayHaveDispatched => {
|
||||
admit()?;
|
||||
ipc::Operation::Execute {
|
||||
generation: intent.connection_generation,
|
||||
request: intent.request.clone(),
|
||||
}
|
||||
}
|
||||
intent::Phase::Prepared => anyhow::bail!("SYSTEM_INTENT_NOT_DURABLE"),
|
||||
};
|
||||
let call = ipc::Call::new(&client.pin, operation);
|
||||
let reply = ipc::exchange(&client.pin, &call, timeout)?;
|
||||
client.check_pin()?;
|
||||
let outcome = match reply.body {
|
||||
ipc::ReplyBody::Executed { outcome }
|
||||
| ipc::ReplyBody::Abandoned { outcome } => outcome,
|
||||
_ => anyhow::bail!("SYSTEM_HELPER_DELEGATION_DENIED"),
|
||||
};
|
||||
checked_outcome(&job, &outcome)?;
|
||||
// A reply is never the consumption/mirror witness. Every outcome,
|
||||
// including a successful reply, requires a fresh raw snapshot.
|
||||
Ok::<_, anyhow::Error>(())
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.context("SYSTEM_HELPER_REPLY_UNKNOWN")??
|
||||
}
|
||||
pub async fn resume_denial(&self, intent: intent::Intent) -> Result<()> {
|
||||
let store = intent::Store::open(false)?;
|
||||
let (retained, phase) = store.load(intent.id)?;
|
||||
ensure!(
|
||||
phase == intent::Phase::KnownNoDispatch && retained.digest()? == intent.digest()?,
|
||||
"SYSTEM_DENIAL_RESUME_BINDING"
|
||||
);
|
||||
let operation = ipc::Operation::Abandon {
|
||||
anchor: Box::new(intent.anchor.clone()),
|
||||
request: intent.request.clone(),
|
||||
intent_id: intent.id,
|
||||
intent_sha256: intent.digest()?,
|
||||
};
|
||||
// Receipt is persisted by the runtime before this method. A timeout
|
||||
// cannot release the blocking lane or cause another Abandon this process.
|
||||
let reply = tokio::time::timeout(Duration::from_secs(5), self.call(operation))
|
||||
.await
|
||||
.context("SYSTEM_ABANDON_REPLY_UNKNOWN")??;
|
||||
ensure!(
|
||||
matches!(reply.body, ipc::ReplyBody::Abandoned { .. }),
|
||||
"SYSTEM_ABANDON_DENIED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
pub async fn snapshot(&self, generation: u64, boot: uuid::Uuid) -> Result<ControlSnapshot> {
|
||||
let reply = self.call(ipc::Operation::Snapshot { generation }).await?;
|
||||
let ipc::ReplyBody::Snapshot { snapshot } = reply.body else {
|
||||
anyhow::bail!("HELPER_SNAPSHOT_DENIED");
|
||||
};
|
||||
ensure!(snapshot.boot_id == boot, "HELPER_BOOT_MISMATCH");
|
||||
Ok(map_snapshot(&snapshot))
|
||||
Ok(ControlSnapshot {
|
||||
raw: snapshot,
|
||||
no_start_anchor: None,
|
||||
})
|
||||
}
|
||||
pub async fn apply(
|
||||
&self,
|
||||
response: &SyncResponse,
|
||||
request_bytes: Vec<u8>,
|
||||
boot: uuid::Uuid,
|
||||
) -> Result<HelperSnapshot> {
|
||||
) -> Result<ControlSnapshot> {
|
||||
let reply = self
|
||||
.call(ipc::Operation::Apply {
|
||||
response: rmm_agent_core::helper_reconcile::SignedResponse {
|
||||
@@ -158,14 +408,112 @@ impl Client {
|
||||
request_bytes,
|
||||
})
|
||||
.await?;
|
||||
let ipc::ReplyBody::Applied { snapshot, .. } = reply.body else {
|
||||
let ipc::ReplyBody::Applied {
|
||||
snapshot,
|
||||
no_start_anchor,
|
||||
..
|
||||
} = reply.body
|
||||
else {
|
||||
anyhow::bail!("HELPER_APPLY_DENIED");
|
||||
};
|
||||
ensure!(snapshot.boot_id == boot, "HELPER_BOOT_MISMATCH");
|
||||
Ok(map_snapshot(&snapshot))
|
||||
if let Some(anchor) = &no_start_anchor {
|
||||
ensure!(
|
||||
anchor.journal_id() == snapshot.journal_id
|
||||
&& anchor.checkpoint() == snapshot.checkpoint
|
||||
&& Some(anchor.decision_id()) == snapshot.previous_decision_id
|
||||
&& anchor.boot_id() == boot,
|
||||
"SYSTEM_ANCHOR_SNAPSHOT_BINDING"
|
||||
);
|
||||
}
|
||||
Ok(ControlSnapshot {
|
||||
raw: snapshot,
|
||||
no_start_anchor: no_start_anchor.map(|a| *a),
|
||||
})
|
||||
}
|
||||
}
|
||||
fn map_snapshot(s: &Snapshot) -> HelperSnapshot {
|
||||
fn verified_mirror_row(
|
||||
row: &rmm_agent_core::helper_reconcile::LocalAttempt,
|
||||
job: &JobPayload,
|
||||
permit: &rmm_domain::jobs::ExecutionPermitPayload,
|
||||
request: &Request,
|
||||
boot: uuid::Uuid,
|
||||
) -> Result<Outcome> {
|
||||
use rmm_domain::{
|
||||
jobs::hash,
|
||||
reconciliation::{ReconciledOutcome, canonical_outcome_bytes},
|
||||
};
|
||||
ensure!(
|
||||
row.job_id == job.job_id
|
||||
&& row.attempt_id == job.attempt_id
|
||||
&& row.job_payload_hash == permit.job_payload_hash
|
||||
&& row.grant_id == permit.grant_id
|
||||
&& row.authorization_id == permit.authorization_id
|
||||
&& row.plan_hash == permit.plan_hash
|
||||
&& row.action == permit.action
|
||||
&& row.execution_identity == "system"
|
||||
&& row.original_boot_id == boot
|
||||
&& row.original_request_sha256 == hash(&serde_json::to_vec(request)?)
|
||||
&& row.consumption == "consumed"
|
||||
&& !row.outcome_bytes.is_empty(),
|
||||
"SYSTEM_MIRROR_ORIGINAL_TUPLE"
|
||||
);
|
||||
let canonical: ReconciledOutcome = serde_json::from_slice(&row.outcome_bytes)?;
|
||||
ensure!(
|
||||
canonical_outcome_bytes(&canonical)? == row.outcome_bytes
|
||||
&& hash(&row.outcome_bytes) == row.outcome_sha256
|
||||
&& canonical.state == row.state,
|
||||
"SYSTEM_MIRROR_CANONICAL_OUTCOME"
|
||||
);
|
||||
let outcome: Outcome = serde_json::from_slice(&row.outcome_bytes)?;
|
||||
checked_outcome(job, &outcome)?;
|
||||
Ok(outcome)
|
||||
}
|
||||
fn system_shape(job: &JobPayload) -> bool {
|
||||
job.execution_identity == "system"
|
||||
&& (1..=120).contains(&job.max_runtime_seconds)
|
||||
&& (1..=65536).contains(&job.max_output_bytes)
|
||||
&& matches!(&job.action,Action::SetServiceState{unit,state} if unit.starts_with("RmmDiscoveryFixture")&&unit.len()>"RmmDiscoveryFixture".len()&&unit.len()<=80&&unit.bytes().all(|b|b.is_ascii_alphanumeric()||b"_-".contains(&b))&&matches!(state.as_str(),"running"|"stopped"))
|
||||
}
|
||||
fn execution_timeout(seconds: u32) -> Result<Duration> {
|
||||
ensure!(
|
||||
(1..=120).contains(&seconds),
|
||||
"SYSTEM_EXECUTION_TIMEOUT_BOUND"
|
||||
);
|
||||
Ok(Duration::from_secs(u64::from(seconds) + 5))
|
||||
}
|
||||
pub(super) fn checked_outcome(job: &JobPayload, outcome: &Outcome) -> Result<()> {
|
||||
ensure!(
|
||||
matches!(
|
||||
outcome.state.as_str(),
|
||||
"succeeded" | "failed" | "cancelled" | "expired" | "unknown"
|
||||
) && !outcome.reboot_required
|
||||
&& outcome
|
||||
.stdout
|
||||
.len()
|
||||
.checked_add(outcome.stderr.len())
|
||||
.is_some_and(|n| n <= job.max_output_bytes as usize),
|
||||
"SYSTEM_EXECUTE_OUTCOME_SCOPE"
|
||||
);
|
||||
if outcome.state == "succeeded" {
|
||||
let Action::SetServiceState { unit, state } = &job.action else {
|
||||
anyhow::bail!("SYSTEM_TYPED_SERVICE_REQUIRED");
|
||||
};
|
||||
let wanted = if state == "running" { 4 } else { 1 };
|
||||
ensure!(
|
||||
outcome.exit_code == Some(0)
|
||||
&& !outcome.output_truncated
|
||||
&& outcome.evidence["postconditions_passed"] == true
|
||||
&& outcome.evidence["provider"] == "windows.scm/1"
|
||||
&& outcome.evidence["service"] == unit.as_str()
|
||||
&& outcome.evidence["requested_state"] == wanted
|
||||
&& outcome.evidence["final_status"]["state"] == wanted,
|
||||
"SYSTEM_SERVICE_POSTCONDITION_BINDING"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn map_snapshot(s: &Snapshot) -> HelperSnapshot {
|
||||
HelperSnapshot {
|
||||
journal_id: s.journal_id,
|
||||
checkpoint: s.checkpoint,
|
||||
@@ -221,3 +569,156 @@ fn write_current_probe(state: &Path, value: &serde_json::Value) -> Result<()> {
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn system_route_binds_original_request_shape_budget_and_reply_scope() {
|
||||
let id = uuid::Uuid::new_v4();
|
||||
let now = chrono::Utc::now();
|
||||
let mut job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: id,
|
||||
attempt_id: id,
|
||||
action: Action::SetServiceState {
|
||||
unit: "RmmDiscoveryFixtureService".into(),
|
||||
state: "running".into(),
|
||||
},
|
||||
execution_identity: "system".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::seconds(60),
|
||||
max_runtime_seconds: 30,
|
||||
max_output_bytes: 1024,
|
||||
};
|
||||
assert!(system_shape(&job));
|
||||
for identity in ["root", "agent_service"] {
|
||||
job.execution_identity = identity.into();
|
||||
assert!(!system_shape(&job));
|
||||
}
|
||||
job.execution_identity = "system".into();
|
||||
job.action = Action::SetServiceState {
|
||||
unit: "RmmDiscoveryFixtureService".into(),
|
||||
state: "restarted".into(),
|
||||
};
|
||||
assert!(!system_shape(&job));
|
||||
job.action = Action::SetServiceState {
|
||||
unit: "RmmDiscoveryWindows".into(),
|
||||
state: "stopped".into(),
|
||||
};
|
||||
assert!(!system_shape(&job));
|
||||
assert_eq!(execution_timeout(120).unwrap(), Duration::from_secs(125));
|
||||
assert!(execution_timeout(0).is_err());
|
||||
assert!(execution_timeout(121).is_err());
|
||||
let mut outcome = Outcome::unknown("LOST_REPLY");
|
||||
assert!(checked_outcome(&job, &outcome).is_ok());
|
||||
outcome.reboot_required = true;
|
||||
assert!(checked_outcome(&job, &outcome).is_err());
|
||||
outcome.reboot_required = false;
|
||||
outcome.stdout = vec![0; 1025];
|
||||
assert!(checked_outcome(&job, &outcome).is_err());
|
||||
job.action = Action::SetServiceState {
|
||||
unit: "RmmDiscoveryFixtureService".into(),
|
||||
state: "running".into(),
|
||||
};
|
||||
outcome.stdout.clear();
|
||||
outcome.state = "succeeded".into();
|
||||
outcome.exit_code = Some(0);
|
||||
assert!(checked_outcome(&job, &outcome).is_err());
|
||||
outcome.evidence = json!({"postconditions_passed":true,"provider":"windows.scm/1","service":"RmmDiscoveryFixtureService","requested_state":4,"final_status":{"state":4}});
|
||||
assert!(checked_outcome(&job, &outcome).is_ok());
|
||||
outcome.evidence["final_status"]["state"] = json!(1);
|
||||
assert!(checked_outcome(&job, &outcome).is_err());
|
||||
let request = Request {
|
||||
job_bytes: vec![b' ', b'{', b'}', b'\n'],
|
||||
permit_bytes: vec![0, 1, 2, 255],
|
||||
signature: vec![7; 64],
|
||||
plan_hash: "ab".repeat(32),
|
||||
authorization_id: id,
|
||||
};
|
||||
let operation = ipc::Operation::Execute {
|
||||
generation: 73,
|
||||
request: request.clone(),
|
||||
};
|
||||
let ipc::Operation::Execute {
|
||||
generation,
|
||||
request: retained,
|
||||
} = serde_json::from_slice(&serde_json::to_vec(&operation).unwrap()).unwrap()
|
||||
else {
|
||||
panic!("typed execute required")
|
||||
};
|
||||
assert_eq!(generation, 73);
|
||||
assert_eq!(
|
||||
serde_json::to_vec(&retained).unwrap(),
|
||||
serde_json::to_vec(&request).unwrap()
|
||||
);
|
||||
// Pure mirror-binding negatives, not a signature/OS authentication proof.
|
||||
let permit = rmm_domain::jobs::ExecutionPermitPayload {
|
||||
domain: "rmm.execution-permit".into(),
|
||||
schema_version: 1,
|
||||
grant_id: id,
|
||||
authorization_id: id,
|
||||
tenant_id: id,
|
||||
asset_id: id,
|
||||
agent_id: id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: id,
|
||||
attempt_id: id,
|
||||
plan_hash: request.plan_hash.clone(),
|
||||
job_payload_hash: rmm_domain::jobs::hash(&request.job_bytes),
|
||||
action: "set_service_state".into(),
|
||||
execution_identity: "system".into(),
|
||||
not_before: now,
|
||||
start_before: now + chrono::Duration::seconds(60),
|
||||
max_runtime_seconds: 30,
|
||||
key_id: "cd".repeat(32),
|
||||
};
|
||||
let canonical: rmm_domain::reconciliation::ReconciledOutcome = serde_json::from_value(
|
||||
serde_json::to_value(Outcome::unknown("RETAINED_UNKNOWN")).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let bytes = rmm_domain::reconciliation::canonical_outcome_bytes(&canonical).unwrap();
|
||||
let row = rmm_agent_core::helper_reconcile::LocalAttempt {
|
||||
job_id: id,
|
||||
attempt_id: id,
|
||||
job_payload_hash: permit.job_payload_hash.clone(),
|
||||
grant_id: id,
|
||||
authorization_id: id,
|
||||
plan_hash: permit.plan_hash.clone(),
|
||||
action: permit.action.clone(),
|
||||
execution_identity: "system".into(),
|
||||
revision: 1,
|
||||
original_boot_id: id,
|
||||
consumption: "consumed".into(),
|
||||
state: "unknown".into(),
|
||||
outcome_sha256: rmm_domain::jobs::hash(&bytes),
|
||||
outcome_bytes: bytes,
|
||||
original_request_sha256: rmm_domain::jobs::hash(&serde_json::to_vec(&request).unwrap()),
|
||||
};
|
||||
assert!(verified_mirror_row(&row, &job, &permit, &request, id).is_ok());
|
||||
for field in [
|
||||
"authorization_id",
|
||||
"original_request_sha256",
|
||||
"original_boot_id",
|
||||
"outcome_sha256",
|
||||
] {
|
||||
let mut changed = serde_json::to_value(&row).unwrap();
|
||||
changed[field] = if field.ends_with("sha256") {
|
||||
json!("ef".repeat(32))
|
||||
} else {
|
||||
json!(uuid::Uuid::new_v4())
|
||||
};
|
||||
let changed = serde_json::from_value(changed).unwrap();
|
||||
assert!(
|
||||
verified_mirror_row(&changed, &job, &permit, &request, id).is_err(),
|
||||
"{field}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,10 @@ use fs2::FileExt;
|
||||
use rmm_agent_core::{
|
||||
auth::{Binding, HelperIdentity, Outcome, Request},
|
||||
crypto::DalekVerifier,
|
||||
helper_reconcile::{BootSource, ExecutorLedgerGuard, HelperSync},
|
||||
helper_reconcile::{
|
||||
BootSource, ExecutorLedgerGuard, HelperSync, NoDispatchObservation, NoStartAnchor,
|
||||
NoStartGuard,
|
||||
},
|
||||
ledger,
|
||||
};
|
||||
use rmm_domain::jobs::{Action, ExecutionPermitPayload, JobPayload, hash};
|
||||
@@ -275,6 +278,48 @@ impl ExecutorLedgerGuard for LedgerGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// All data other than the immutable intent reference comes from held local
|
||||
/// handles. The IPC caller cannot manufacture a NoDispatchObservation.
|
||||
struct DenialGuard<'a> {
|
||||
ledger: &'a LedgerGuard,
|
||||
pin: &'a Pin,
|
||||
intent: Option<(Uuid, &'a str)>,
|
||||
}
|
||||
impl ExecutorLedgerGuard for DenialGuard<'_> {
|
||||
fn check(&self, db: &Connection) -> Result<()> {
|
||||
ipc::require_self(&self.pin.helper)?;
|
||||
self.ledger.check(db)
|
||||
}
|
||||
}
|
||||
impl NoStartGuard for DenialGuard<'_> {
|
||||
fn live_instance_sha256(&self) -> Result<String> {
|
||||
let pid = std::process::id();
|
||||
let peer = ipc::PeerProof::for_service(&self.pin.helper, pid)?;
|
||||
Ok(hash(&serde_json::to_vec(&(
|
||||
peer.identity(),
|
||||
self.ledger.ledger.identity,
|
||||
self.ledger.lock.identity,
|
||||
))?))
|
||||
}
|
||||
fn verify_no_dispatch(
|
||||
&self,
|
||||
original: &Request,
|
||||
anchor: &NoStartAnchor,
|
||||
) -> Result<NoDispatchObservation> {
|
||||
let (id, digest) = self
|
||||
.intent
|
||||
.context("SYSTEM_NO_DISPATCH_REFERENCE_REQUIRED")?;
|
||||
let mut store = crate::helper_runtime::intent::Store::open(false)?;
|
||||
let (intent, _) = store.load(id)?;
|
||||
ensure!(
|
||||
intent.policy_sha256 == self.pin.policy_sha256
|
||||
&& intent.helper_binary_sha256 == self.pin.helper.binary_sha256,
|
||||
"SYSTEM_NO_DISPATCH_PIN_BINDING"
|
||||
);
|
||||
store.verify_no_dispatch(id, digest, original, anchor)
|
||||
}
|
||||
}
|
||||
|
||||
struct ServiceHandle(SC_HANDLE);
|
||||
impl Drop for ServiceHandle {
|
||||
fn drop(&mut self) {
|
||||
@@ -727,6 +772,7 @@ pub struct SystemHelper {
|
||||
clock: NativeClock,
|
||||
boot: WindowsBoot,
|
||||
sync: Option<HelperSync<DalekVerifier>>,
|
||||
last_diagnostic: Option<Instant>,
|
||||
}
|
||||
impl SystemHelper {
|
||||
pub fn open() -> Result<Self> {
|
||||
@@ -749,11 +795,59 @@ impl SystemHelper {
|
||||
clock,
|
||||
boot,
|
||||
sync,
|
||||
last_diagnostic: None,
|
||||
})
|
||||
}
|
||||
pub fn pin(&self) -> &Pin {
|
||||
&self.pin
|
||||
}
|
||||
/// One bounded private bootstrap diagnostic, never inventory or payload.
|
||||
/// Diagnostic failure is intentionally ignored by the service loop; the
|
||||
/// original security failure still closes Sync and disconnects the pipe.
|
||||
fn record_pipe_error(&mut self, error: &anyhow::Error) -> Result<()> {
|
||||
if self
|
||||
.last_diagnostic
|
||||
.is_some_and(|at| at.elapsed() < Duration::from_secs(30))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
self.last_diagnostic = Some(Instant::now());
|
||||
self.recheck()?;
|
||||
let path = Path::new(STATE_ROOT).join("pipe-last-error.json");
|
||||
let pending = Path::new(STATE_ROOT).join(".pipe-last-error.pending");
|
||||
let writers = vec![self.settings.helper.service_sid.clone()];
|
||||
if path.try_exists()? {
|
||||
let _old = PinnedFile::open(&path, false, &[], &writers, false)?;
|
||||
}
|
||||
let diagnostic = ipc::diagnostic(error);
|
||||
let bytes = serde_json::to_vec(
|
||||
&serde_json::json!({"pid":std::process::id(),"observed_at_utc":chrono::Utc::now(),"diagnostic":diagnostic}),
|
||||
)?;
|
||||
ensure!(bytes.len() <= 1024, "SYSTEM_DIAGNOSTIC_BOUND");
|
||||
use std::io::Write;
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&pending)?;
|
||||
let identity = ipc::file_identity(&file)?;
|
||||
let result = (|| -> Result<()> {
|
||||
{
|
||||
let _check = PinnedFile::open_read_shared(&pending, false, &[], &writers)?;
|
||||
}
|
||||
file.write_all(&bytes)?;
|
||||
file.sync_all()?;
|
||||
Ok(())
|
||||
})();
|
||||
drop(file);
|
||||
result?;
|
||||
std::fs::rename(&pending, &path)?;
|
||||
let saved = PinnedFile::open(&path, false, &[], &writers, false)?;
|
||||
ensure!(
|
||||
saved.identity == identity,
|
||||
"SYSTEM_DIAGNOSTIC_FILE_REPLACED"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn recheck(&mut self) -> Result<()> {
|
||||
ipc::require_self(&self.settings.helper)?;
|
||||
for file in &mut self.files {
|
||||
@@ -829,10 +923,33 @@ impl SystemHelper {
|
||||
&self.clock,
|
||||
&self.boot,
|
||||
)?;
|
||||
let no_start_anchor = if value.ready && self.settings.enable_effects {
|
||||
let guard = DenialGuard {
|
||||
ledger: &self.guard,
|
||||
pin: &self.pin,
|
||||
intent: None,
|
||||
};
|
||||
let generation = self
|
||||
.sync
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.active_generation()
|
||||
.context("SYSTEM_READY_GENERATION_REQUIRED")?;
|
||||
Some(Box::new(self.sync.as_ref().unwrap().no_start_anchor(
|
||||
&self.db,
|
||||
&guard,
|
||||
generation,
|
||||
&self.clock,
|
||||
&self.boot,
|
||||
)?))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(ReplyBody::Applied {
|
||||
snapshot: value.snapshot,
|
||||
unlocked_grants: value.unlocked_grants,
|
||||
ready: value.ready && self.settings.enable_effects,
|
||||
no_start_anchor,
|
||||
})
|
||||
}
|
||||
Operation::Execute {
|
||||
@@ -841,6 +958,40 @@ impl SystemHelper {
|
||||
} => Ok(ReplyBody::Executed {
|
||||
outcome: self.execute(*generation, request, peer)?,
|
||||
}),
|
||||
Operation::Abandon {
|
||||
anchor,
|
||||
request,
|
||||
intent_id,
|
||||
intent_sha256,
|
||||
} => {
|
||||
ensure!(
|
||||
!intent_id.is_nil() && ipc::digest(intent_sha256),
|
||||
"SYSTEM_NO_DISPATCH_REFERENCE_REQUIRED"
|
||||
);
|
||||
let job = self.settings.binding.job(&request.job_bytes)?;
|
||||
ensure!(
|
||||
job.execution_identity == "system"
|
||||
&& job.max_runtime_seconds <= self.settings.max_runtime_seconds
|
||||
&& matches!(&job.action,Action::SetServiceState{unit,state} if matches!(state.as_str(),"running"|"stopped")&&self.settings.allowed_services.iter().any(|r|r.name==*unit)),
|
||||
"SYSTEM_ABANDON_POLICY_DENIED"
|
||||
);
|
||||
self.sync()?;
|
||||
let guard = DenialGuard {
|
||||
ledger: &self.guard,
|
||||
pin: &self.pin,
|
||||
intent: Some((*intent_id, intent_sha256)),
|
||||
};
|
||||
Ok(ReplyBody::Abandoned {
|
||||
outcome: self.sync.as_mut().unwrap().abandon_no_start(
|
||||
&mut self.db,
|
||||
&guard,
|
||||
anchor,
|
||||
request,
|
||||
&self.clock,
|
||||
&self.boot,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
};
|
||||
let response = response?;
|
||||
peer.check()?;
|
||||
@@ -1028,19 +1179,22 @@ unsafe extern "system" fn service_main(_: u32, _: *mut *mut u16) {
|
||||
match server.receive() {
|
||||
Ok(Some(mut incoming)) => {
|
||||
let reply = helper.handle(&incoming.call, &mut incoming.peer);
|
||||
if server.reply(&reply).is_err()
|
||||
&& let Some(sync) = &mut helper.sync
|
||||
{
|
||||
sync.close();
|
||||
if let Err(error) = server.reply(&reply) {
|
||||
if let Some(sync) = &mut helper.sync {
|
||||
sync.close();
|
||||
}
|
||||
server.disconnect();
|
||||
let _diagnostic = helper.record_pipe_error(&error);
|
||||
}
|
||||
server.disconnect();
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(_) => {
|
||||
Err(error) => {
|
||||
if let Some(sync) = &mut helper.sync {
|
||||
sync.close();
|
||||
}
|
||||
server.disconnect();
|
||||
let _diagnostic = helper.record_pipe_error(&error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,14 +135,20 @@ impl SyncClock for JobClock {
|
||||
#[derive(Default)]
|
||||
struct EffectFence {
|
||||
generation: AtomicU64,
|
||||
helper_generation: AtomicU64,
|
||||
deadline: AtomicI64,
|
||||
}
|
||||
impl EffectFence {
|
||||
fn close(&self) {
|
||||
self.generation.store(0, Ordering::SeqCst);
|
||||
self.helper_generation.store(0, Ordering::SeqCst);
|
||||
}
|
||||
fn allows(&self, generation: u64, clock: &impl SyncClock) -> bool {
|
||||
self.generation.load(Ordering::SeqCst) == generation
|
||||
self.allows_identity(generation, false, clock)
|
||||
}
|
||||
fn allows_identity(&self, generation: u64, helper: bool, clock: &impl SyncClock) -> bool {
|
||||
(!helper || self.helper_generation.load(Ordering::SeqCst) == generation)
|
||||
&& self.generation.load(Ordering::SeqCst) == generation
|
||||
&& generation > 0
|
||||
&& clock.now().is_ok_and(|now| {
|
||||
(now + chrono::Duration::seconds(2)).timestamp_millis()
|
||||
@@ -151,6 +157,48 @@ impl EffectFence {
|
||||
}
|
||||
}
|
||||
type Completion = (JobOffer, ExecutionPermitPayload, Outcome);
|
||||
#[cfg(feature = "system-helper")]
|
||||
fn needs_bound_outcome(
|
||||
db: &Connection,
|
||||
bound: &rmm_agent_windows::helper_runtime::BoundOutcome,
|
||||
) -> Result<bool> {
|
||||
use rmm_domain::reconciliation::{ReconciledOutcome, canonical_outcome_bytes};
|
||||
// Archiving is durable retention only: no ACK, event deletion or unlock.
|
||||
reconcile::archive_events(db)?;
|
||||
let (state, revision, grant): (String, u64, String) = db.query_row(
|
||||
"SELECT state,revision,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![bound.job_id.to_string(), bound.attempt_id.to_string()],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)?;
|
||||
let bytes:Option<Vec<u8>>=db.query_row("SELECT envelope FROM reconcile_results WHERE job=?1 AND attempt=?2 ORDER BY revision DESC LIMIT 1",rusqlite::params![bound.job_id.to_string(),bound.attempt_id.to_string()],|r|r.get(0)).optional()?;
|
||||
if state == bound.outcome.state
|
||||
&& grant == bound.grant_id.to_string()
|
||||
&& let Some(bytes) = bytes
|
||||
{
|
||||
let envelope = Envelope::decode(bytes.as_slice())?;
|
||||
if let Some(Payload::JobResult(r)) = envelope.payload
|
||||
&& r.revision == revision
|
||||
&& r.grant_id == grant
|
||||
{
|
||||
let saved = ReconciledOutcome {
|
||||
state: r.state,
|
||||
stdout: r.stdout,
|
||||
stderr: r.stderr,
|
||||
exit_code: r.exit_code,
|
||||
output_truncated: r.output_truncated,
|
||||
reboot_required: r.reboot_required,
|
||||
evidence: serde_json::from_str(&r.evidence_json)?,
|
||||
error_code: r.error_code,
|
||||
};
|
||||
let actual: ReconciledOutcome =
|
||||
serde_json::from_value(serde_json::to_value(&bound.outcome)?)?;
|
||||
if canonical_outcome_bytes(&saved)? == canonical_outcome_bytes(&actual)? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
struct Running {
|
||||
offer: JobOffer,
|
||||
grant: ExecutionPermitPayload,
|
||||
@@ -338,26 +386,40 @@ fn open_ledger(directory: &Path, binding: &Binding) -> Result<(Connection, fs::F
|
||||
Ok((db, file))
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
type MirrorCompletion = (
|
||||
Connection,
|
||||
Result<(
|
||||
rmm_agent_windows::helper_runtime::ControlSnapshot,
|
||||
Vec<rmm_agent_windows::helper_runtime::BoundOutcome>,
|
||||
)>,
|
||||
);
|
||||
#[cfg(feature = "system-helper")]
|
||||
enum HelperControl {
|
||||
Snapshot(
|
||||
u64,
|
||||
tokio::task::JoinHandle<Result<reconcile::HelperSnapshot>>,
|
||||
tokio::task::JoinHandle<Result<rmm_agent_windows::helper_runtime::ControlSnapshot>>,
|
||||
),
|
||||
Apply(
|
||||
u64,
|
||||
tokio::task::JoinHandle<Result<reconcile::HelperSnapshot>>,
|
||||
tokio::task::JoinHandle<Result<rmm_agent_windows::helper_runtime::ControlSnapshot>>,
|
||||
),
|
||||
Mirror(u64, tokio::task::JoinHandle<MirrorCompletion>),
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
impl HelperControl {
|
||||
fn finished(&self) -> bool {
|
||||
match self {
|
||||
Self::Snapshot(_, t) | Self::Apply(_, t) => t.is_finished(),
|
||||
Self::Mirror(_, t) => t.is_finished(),
|
||||
}
|
||||
}
|
||||
fn abort(self) {
|
||||
fn abort(self) -> Option<Self> {
|
||||
match self {
|
||||
Self::Snapshot(_, t) | Self::Apply(_, t) => t.abort(),
|
||||
Self::Snapshot(_, t) | Self::Apply(_, t) => {
|
||||
t.abort();
|
||||
None
|
||||
}
|
||||
mirror @ Self::Mirror(..) => Some(mirror),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -389,6 +451,8 @@ struct Agent {
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_control: Option<HelperControl>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_anchor: Option<rmm_agent_core::helper_reconcile::NoStartAnchor>,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_checked: bool,
|
||||
#[cfg(feature = "system-helper")]
|
||||
helper_refused: bool,
|
||||
@@ -408,8 +472,9 @@ impl Agent {
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
self.prepared_request = None;
|
||||
self.helper_anchor = None;
|
||||
if let Some(control) = self.helper_control.take() {
|
||||
control.abort();
|
||||
self.helper_control = control.abort();
|
||||
}
|
||||
}
|
||||
self.deferred_permit = None;
|
||||
@@ -446,15 +511,69 @@ impl Agent {
|
||||
HelperControl::Snapshot(expected, task) => {
|
||||
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
|
||||
let snapshot = task.await??;
|
||||
self.send_sync(channel, generation, Some(&snapshot)).await?;
|
||||
let helper = self
|
||||
.helper
|
||||
.clone()
|
||||
.context("WINDOWS_HELPER_NOT_CONFIGURED")?;
|
||||
let observed_at = self.clock.now()?;
|
||||
let mut ledger = self.ledger.take().context("LEDGER_REQUIRED")?;
|
||||
self.helper_control = Some(HelperControl::Mirror(
|
||||
generation,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = helper
|
||||
.mirror(&snapshot.raw, &mut ledger, observed_at)
|
||||
.map(|unknowns| (snapshot, unknowns));
|
||||
(ledger, result)
|
||||
}),
|
||||
));
|
||||
}
|
||||
HelperControl::Mirror(expected, task) => {
|
||||
// Always restore ownership, including after reconnect. A
|
||||
// stale generation may discard its snapshot, never its DB.
|
||||
let (ledger, result) = task.await?;
|
||||
self.ledger = Some(ledger);
|
||||
let (snapshot, unknowns) = result?;
|
||||
for bound in unknowns {
|
||||
if needs_bound_outcome(self.journal.connection(), &bound)? {
|
||||
let bytes: Vec<u8> = self.journal.connection().query_row(
|
||||
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![
|
||||
bound.job_id.to_string(),
|
||||
bound.attempt_id.to_string()
|
||||
],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
self.record(
|
||||
generation,
|
||||
&JobOffer::decode(bytes.as_slice())?,
|
||||
&bound.grant_id.to_string(),
|
||||
bound.outcome,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
if expected != generation {
|
||||
self.close_gate();
|
||||
return Ok(());
|
||||
}
|
||||
self.send_sync(
|
||||
channel,
|
||||
generation,
|
||||
Some(&rmm_agent_windows::helper_runtime::map_snapshot(
|
||||
&snapshot.raw,
|
||||
)),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
HelperControl::Apply(expected, task) => {
|
||||
ensure!(expected == generation, "HELPER_CHANNEL_CHANGED");
|
||||
let snapshot = task.await??;
|
||||
self.helper_anchor = snapshot.no_start_anchor;
|
||||
self.sync
|
||||
.as_mut()
|
||||
.context("SYNC_NOT_CONFIGURED")?
|
||||
.confirm_helper(&snapshot)?;
|
||||
.confirm_helper(&rmm_agent_windows::helper_runtime::map_snapshot(
|
||||
&snapshot.raw,
|
||||
))?;
|
||||
self.open_fence(generation)?;
|
||||
}
|
||||
}
|
||||
@@ -493,6 +612,7 @@ impl Agent {
|
||||
&& self.helper_control.is_none()
|
||||
&& !self.prepared
|
||||
&& self.running.is_none()
|
||||
&& !self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get::<_,bool>(0))?
|
||||
&& self.helper_probe_at.elapsed() >= Duration::from_secs(30)
|
||||
{
|
||||
let agent = self.identity.agent_id.clone();
|
||||
@@ -516,14 +636,45 @@ impl Agent {
|
||||
})
|
||||
}
|
||||
fn ready(&self, generation: u64) -> bool {
|
||||
self.ready_identity(generation, false)
|
||||
}
|
||||
fn ready_identity(&self, generation: u64, helper: bool) -> bool {
|
||||
#[cfg(feature = "system-helper")]
|
||||
if helper
|
||||
&& !self.helper_anchor.as_ref().is_some_and(|a| {
|
||||
a.connection_generation() == generation && a.boot_id().to_string() == self.boot
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
self.config.enable_jobs
|
||||
&& !self.helper_control_pending()
|
||||
&& self.channel(generation).is_ok_and(|channel| {
|
||||
self.sync
|
||||
.as_ref()
|
||||
.is_some_and(|sync| sync.can_start(&channel, false, &self.clock))
|
||||
.is_some_and(|sync| sync.can_start(&channel, helper, &self.clock))
|
||||
})
|
||||
}
|
||||
fn supported_job(&self, job: &rmm_domain::jobs::JobPayload) -> bool {
|
||||
if native::require_local_service().is_err() {
|
||||
return false;
|
||||
}
|
||||
if rmm_agent_windows::actions::supported(job) {
|
||||
return true;
|
||||
}
|
||||
#[cfg(feature = "system-helper")]
|
||||
{
|
||||
Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file()
|
||||
&& self
|
||||
.helper
|
||||
.as_ref()
|
||||
.is_some_and(|helper| helper.supports_system(job))
|
||||
}
|
||||
#[cfg(not(feature = "system-helper"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
fn capabilities(&self) -> Vec<String> {
|
||||
let mut caps: Vec<String> = CAPABILITIES.iter().map(|c| c.to_string()).collect();
|
||||
if self.config.enable_jobs
|
||||
@@ -559,6 +710,72 @@ impl Agent {
|
||||
let Some(binding) = self.binding.clone() else {
|
||||
return Ok(());
|
||||
};
|
||||
#[cfg(feature = "system-helper")]
|
||||
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
||||
let mut intents = rmm_agent_windows::helper_runtime::intent::Store::open(true)?;
|
||||
let pending = intents.pending()?;
|
||||
if let Some(intent) = pending.first() {
|
||||
let p = intent.permit()?;
|
||||
let incompatible:bool=self.journal.connection().query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying','unknown','awaiting_reboot') AND NOT(job=?1 AND attempt=?2))",rusqlite::params![p.job_id.to_string(),p.attempt_id.to_string()],|r|r.get(0))?;
|
||||
ensure!(!incompatible, "SYSTEM_PENDING_INTENT_CONFLICT");
|
||||
binding.verify_historical_for_denial_with(
|
||||
&intent.request,
|
||||
reconcile::HelperIdentity::System,
|
||||
&rmm_agent_core::crypto::DalekVerifier,
|
||||
)?;
|
||||
let bytes: Vec<u8> = self.journal.connection().query_row(
|
||||
"SELECT offer FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![p.job_id.to_string(), p.attempt_id.to_string()],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let offer = JobOffer::decode(bytes.as_slice())?;
|
||||
ensure!(
|
||||
offer.payload_bytes == intent.request.job_bytes
|
||||
&& offer.plan_hash == intent.request.plan_hash
|
||||
&& offer.authorization_id == intent.request.authorization_id.to_string(),
|
||||
"SYSTEM_PENDING_INTENT_BINDING"
|
||||
);
|
||||
// Even prepared-only intents are never executed after restart.
|
||||
// Snapshot/explicit bound denial recovery is the only route.
|
||||
intents.check()?;
|
||||
let ledger = self.ledger.as_ref().context("LEDGER_REQUIRED")?;
|
||||
let consumed: bool = ledger.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM consumed WHERE grant_id=?1)",
|
||||
[p.grant_id.to_string()],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let actual = if consumed {
|
||||
jobs::recover_interrupted_outcome(
|
||||
ledger,
|
||||
&binding,
|
||||
&offer,
|
||||
&p.grant_id.to_string(),
|
||||
)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(actual) = &actual {
|
||||
let bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
||||
job_id: p.job_id,
|
||||
attempt_id: p.attempt_id,
|
||||
grant_id: p.grant_id,
|
||||
outcome: actual.clone(),
|
||||
};
|
||||
if !needs_bound_outcome(self.journal.connection(), &bound)? {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
self.record(
|
||||
0,
|
||||
&offer,
|
||||
&p.grant_id.to_string(),
|
||||
actual.unwrap_or_else(|| {
|
||||
Outcome::unknown("SYSTEM_DELEGATION_INTERRUPTED_REQUIRES_HELPER_EVIDENCE")
|
||||
}),
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let interrupted = jobs::interrupted(
|
||||
self.journal.connection(),
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
@@ -610,9 +827,38 @@ impl Agent {
|
||||
if let Some(helper) = self.helper.clone() {
|
||||
ensure!(self.helper_control.is_none(), "HELPER_CONTROL_PENDING");
|
||||
let boot = self.channel(generation)?.boot_id;
|
||||
let instance = Uuid::parse_str(&self.instance)?;
|
||||
let clock = self.clock.clone();
|
||||
self.helper_control = Some(HelperControl::Snapshot(
|
||||
generation,
|
||||
tokio::spawn(async move { helper.snapshot(generation, boot).await }),
|
||||
tokio::spawn(async move {
|
||||
let denial_resume = tokio::task::spawn_blocking(move || -> Result<_> {
|
||||
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
||||
use rmm_agent_windows::helper_runtime::intent::{Phase, Store};
|
||||
let mut store = Store::open(true)?;
|
||||
if let Some(intent) = store.pending()?.first() {
|
||||
let (_, phase) = store.load(intent.id)?;
|
||||
if phase == Phase::KnownNoDispatch
|
||||
&& intent.agent_instance != instance
|
||||
&& store.claim_denial_resume(
|
||||
intent.id,
|
||||
instance,
|
||||
boot,
|
||||
clock.now()?,
|
||||
)?
|
||||
{
|
||||
return Ok(Some(intent.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
})
|
||||
.await??;
|
||||
if let Some(intent) = denial_resume {
|
||||
let _denial = helper.resume_denial(intent).await;
|
||||
}
|
||||
helper.snapshot(generation, boot).await
|
||||
}),
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
@@ -694,6 +940,14 @@ impl Agent {
|
||||
.deadline
|
||||
.store(deadline.timestamp_millis(), Ordering::SeqCst);
|
||||
self.fence.generation.store(generation, Ordering::SeqCst);
|
||||
self.fence.helper_generation.store(
|
||||
if self.ready_identity(generation, true) {
|
||||
generation
|
||||
} else {
|
||||
0
|
||||
},
|
||||
Ordering::SeqCst,
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -708,6 +962,19 @@ impl Agent {
|
||||
&& running.grant == grant,
|
||||
"EXECUTOR_COMPLETION_BINDING"
|
||||
);
|
||||
if grant.execution_identity == "system" {
|
||||
// No local consumption or definitive result from an IPC outcome.
|
||||
// Import only a fresh raw helper snapshot before the next Sync.
|
||||
self.record(
|
||||
generation,
|
||||
&offer,
|
||||
&grant.grant_id.to_string(),
|
||||
Outcome::unknown("SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT"),
|
||||
)?;
|
||||
self.running = None;
|
||||
self.close_gate();
|
||||
return Ok(());
|
||||
}
|
||||
ledger::persist_outcome(
|
||||
self.ledger.as_ref().context("LEDGER_REQUIRED")?,
|
||||
&grant,
|
||||
@@ -725,6 +992,9 @@ impl Agent {
|
||||
&outcome,
|
||||
)?;
|
||||
self.running = None;
|
||||
if grant.execution_identity == "system" {
|
||||
self.close_gate();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn permit(&mut self, generation: u64, permit: ExecutionPermit) -> Result<()> {
|
||||
@@ -764,10 +1034,82 @@ impl Agent {
|
||||
return Ok(());
|
||||
}
|
||||
let (job, grant) = verified_window(&binding, &request, &self.clock)?;
|
||||
ensure!(self.supported_job(&job), "UNSUPPORTED_CAPABILITY");
|
||||
let system = job.execution_identity == "system";
|
||||
ensure!(
|
||||
rmm_agent_windows::actions::supported(&job),
|
||||
"UNSUPPORTED_CAPABILITY"
|
||||
self.ready_identity(generation, system),
|
||||
"SYSTEM_HELPER_SYNC_REQUIRED"
|
||||
);
|
||||
let cancelled: bool = self.journal.connection().query_row(
|
||||
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
#[cfg(feature = "system-helper")]
|
||||
if system {
|
||||
use rmm_agent_windows::helper_runtime::intent::{Intent, Store};
|
||||
let helper = self
|
||||
.helper
|
||||
.clone()
|
||||
.context("WINDOWS_HELPER_NOT_CONFIGURED")?;
|
||||
let intent = Intent {
|
||||
schema_version: 1,
|
||||
id: Uuid::new_v4(),
|
||||
request: request.clone(),
|
||||
anchor: self
|
||||
.helper_anchor
|
||||
.clone()
|
||||
.context("SYSTEM_READY_ANCHOR_REQUIRED")?,
|
||||
original_boot_id: Uuid::parse_str(&self.boot)?,
|
||||
agent_instance: Uuid::parse_str(&self.instance)?,
|
||||
connection_generation: generation,
|
||||
recorded_at: self.clock.now()?,
|
||||
policy_sha256: helper.pin_hash().into(),
|
||||
helper_binary_sha256: helper.helper_binary_hash().into(),
|
||||
};
|
||||
let mut store = Store::open(true)?;
|
||||
store.prepare(&intent)?;
|
||||
if cancelled {
|
||||
store.decide(intent.id, false)?;
|
||||
}
|
||||
let cancelled = Arc::new(AtomicBool::new(cancelled));
|
||||
let worker_cancel = cancelled.clone();
|
||||
let clock = self.clock.clone();
|
||||
let fence = self.fence.clone();
|
||||
let completed = self.completed_tx.clone();
|
||||
let work_offer = offer.clone();
|
||||
let work_grant = grant.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
let _result = helper
|
||||
.delegate(intent, move || {
|
||||
ensure!(
|
||||
!worker_cancel.load(Ordering::SeqCst),
|
||||
"CANCELLED_BEFORE_DISPATCH"
|
||||
);
|
||||
ensure!(
|
||||
fence.allows_identity(generation, true, &clock),
|
||||
"FENCED_BEFORE_DISPATCH"
|
||||
);
|
||||
verified_window(&binding, &request, &clock)?;
|
||||
Ok(())
|
||||
})
|
||||
.await;
|
||||
let _ = completed
|
||||
.send((
|
||||
work_offer,
|
||||
work_grant,
|
||||
Outcome::unknown("SYSTEM_DELEGATION_REQUIRES_HELPER_SNAPSHOT"),
|
||||
))
|
||||
.await;
|
||||
});
|
||||
self.running = Some(Running {
|
||||
offer,
|
||||
grant,
|
||||
cancel: cancelled,
|
||||
worker,
|
||||
});
|
||||
return Ok(());
|
||||
}
|
||||
let started_at = self.clock.now()?;
|
||||
let ledger = self.ledger.as_mut().context("LEDGER_REQUIRED")?;
|
||||
ledger::begin_execution(
|
||||
@@ -777,11 +1119,6 @@ impl Agent {
|
||||
Uuid::parse_str(&self.boot)?,
|
||||
started_at,
|
||||
)?;
|
||||
let cancelled: bool = self.journal.connection().query_row(
|
||||
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
||||
rusqlite::params![offer.job_id, offer.attempt_id],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
if cancelled {
|
||||
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
|
||||
ledger::persist_outcome(
|
||||
@@ -1016,7 +1353,7 @@ impl Agent {
|
||||
if let Some(offer)=jobs::next_at(self.journal.connection(),now)? {
|
||||
let job=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.job(&offer.payload_bytes)?;
|
||||
if now>=job.start_before {self.record(generation,&offer,"",jobs::no_start("expired","START_DEADLINE_EXPIRED"))?;needs_sync=true;}
|
||||
else if now-chrono::Duration::seconds(2)>=job.not_before {
|
||||
else if now-chrono::Duration::seconds(2)>=job.not_before && self.ready_identity(generation,job.execution_identity=="system") {
|
||||
jobs::reserve_request(self.journal.connection(),&offer)?;
|
||||
let request=self.envelope(generation,Payload::ExecutionPermitRequest(ExecutionPermitRequest{job_id:offer.job_id,attempt_id:offer.attempt_id,job_payload_hash:offer.job_payload_hash,assignment_version:job.assignment_version}))?;
|
||||
channel.send(&request).await?;
|
||||
@@ -1039,8 +1376,9 @@ impl Agent {
|
||||
ensure!(self.config.enable_jobs,"UNSUPPORTED_EXECUTION_CAPABILITY");
|
||||
let binding=self.binding.as_ref().context("JOB_BINDING_REQUIRED")?.clone();
|
||||
let accepted=self.envelope(generation,Payload::JobAccepted(JobAccepted{job_id:offer.job_id.clone(),attempt_id:offer.attempt_id.clone(),job_payload_hash:offer.job_payload_hash.clone(),journal_proof:rmm_domain::jobs::hash(&offer.encode_to_vec())}))?;
|
||||
let supported=self.supported_job(&binding.job(&offer.payload_bytes)?);
|
||||
let tx=self.journal.connection_mut().transaction()?;
|
||||
let fresh=jobs::accept(&tx,&binding,&offer,rmm_agent_windows::actions::supported)?;
|
||||
let fresh=jobs::accept(&tx,&binding,&offer,|_|supported)?;
|
||||
jobs::event(&tx,&accepted)?;tx.commit()?;needs_sync|=fresh;
|
||||
},
|
||||
Some(Payload::ExecutionPermit(permit))=>self.permit(generation,permit)?,
|
||||
@@ -1049,6 +1387,11 @@ impl Agent {
|
||||
if let Some((bytes,state,grant))=row {
|
||||
self.journal.connection().execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2",rusqlite::params![cancel.job_id,cancel.attempt_id])?;
|
||||
if let Some(running)=&self.running && running.offer.job_id==cancel.job_id && running.offer.attempt_id==cancel.attempt_id {running.cancel.store(true,Ordering::SeqCst);}
|
||||
#[cfg(feature="system-helper")]
|
||||
if Path::new(rmm_agent_windows::helper_runtime::intent::DB).is_file() {
|
||||
let mut store=rmm_agent_windows::helper_runtime::intent::Store::open(true)?;
|
||||
if let Some(intent)=store.pending()?.first() && intent.permit()?.job_id.to_string()==cancel.job_id && intent.permit()?.attempt_id.to_string()==cancel.attempt_id {store.cancel_before_dispatch(intent.id)?;}
|
||||
}
|
||||
if state=="accepted" {self.record(generation,&JobOffer::decode(bytes.as_slice())?,&grant,jobs::no_start("cancelled","CANCELLED_BEFORE_START"))?;needs_sync=true;}
|
||||
}
|
||||
let ack=self.envelope(generation,Payload::Ack(Ack{message_id:message.message_id}))?;channel.send(&ack).await?;
|
||||
@@ -1158,6 +1501,8 @@ fn service_run(stop: &'static AtomicBool) -> Result<()> {
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_control:None,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_anchor:None,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_checked:false,
|
||||
#[cfg(feature="system-helper")]
|
||||
helper_refused:false,
|
||||
@@ -1242,6 +1587,90 @@ pub fn main() -> Result<()> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[cfg(feature = "system-helper")]
|
||||
#[test]
|
||||
fn bound_helper_unknown_repairs_generic_crash_gap_without_ack_or_duplicate_result() {
|
||||
let path = std::env::temp_dir().join(format!("windows-bound-unknown-{}", Uuid::new_v4()));
|
||||
fs::create_dir(&path).unwrap();
|
||||
let journal = Journal::open(&path).unwrap();
|
||||
let db = journal.connection();
|
||||
let job = Uuid::new_v4();
|
||||
let attempt = Uuid::new_v4();
|
||||
let grant = Uuid::new_v4();
|
||||
let offer = JobOffer {
|
||||
job_id: job.to_string(),
|
||||
attempt_id: attempt.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
db.execute("INSERT INTO job_attempts(job,attempt,offer,state,revision,grant_id) VALUES(?1,?2,?3,'unknown',1,?4)",rusqlite::params![offer.job_id,offer.attempt_id,offer.encode_to_vec(),grant.to_string()]).unwrap();
|
||||
let original = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
payload: Some(Payload::JobResult(jobs::result(
|
||||
&offer,
|
||||
1,
|
||||
&grant.to_string(),
|
||||
Outcome::unknown("GENERIC_REPLY_LOST"),
|
||||
))),
|
||||
..Default::default()
|
||||
};
|
||||
jobs::event(db, &original).unwrap();
|
||||
let mut actual = Outcome::unknown("SYSTEM_PREVIOUS_EFFECT_UNRESOLVED");
|
||||
actual.evidence = serde_json::json!({"retained_helper_evidence":true});
|
||||
let bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
||||
job_id: job,
|
||||
attempt_id: attempt,
|
||||
grant_id: grant,
|
||||
outcome: actual.clone(),
|
||||
};
|
||||
// Represents a crash after actual raw-helper outcome persistence but
|
||||
// before the corresponding journal write, even if mirror receipt exists.
|
||||
assert!(needs_bound_outcome(db, &bound).unwrap());
|
||||
assert_eq!(jobs::events(db).unwrap().len(), 1);
|
||||
let revision = jobs::transition(db, &offer, "unknown", &grant.to_string()).unwrap();
|
||||
let repaired = Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
payload: Some(Payload::JobResult(jobs::result(
|
||||
&offer,
|
||||
revision,
|
||||
&grant.to_string(),
|
||||
actual,
|
||||
))),
|
||||
..Default::default()
|
||||
};
|
||||
jobs::event(db, &repaired).unwrap();
|
||||
assert!(!needs_bound_outcome(db, &bound).unwrap());
|
||||
assert_eq!(jobs::events(db).unwrap().len(), 2); // retention is never ACK
|
||||
let mut terminal = bound.outcome.clone();
|
||||
terminal.state = "failed".into();
|
||||
let terminal_bound = rmm_agent_windows::helper_runtime::BoundOutcome {
|
||||
job_id: job,
|
||||
attempt_id: attempt,
|
||||
grant_id: grant,
|
||||
outcome: terminal.clone(),
|
||||
};
|
||||
let revision = jobs::transition(db, &offer, "failed", &grant.to_string()).unwrap();
|
||||
jobs::event(
|
||||
db,
|
||||
&Envelope {
|
||||
message_id: Uuid::new_v4().to_string(),
|
||||
payload: Some(Payload::JobResult(jobs::result(
|
||||
&offer,
|
||||
revision,
|
||||
&grant.to_string(),
|
||||
terminal,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
// Two restarts after the terminal journal committed but before a
|
||||
// mirror receipt may never request terminal -> terminal transitions.
|
||||
assert!(!needs_bound_outcome(db, &terminal_bound).unwrap());
|
||||
assert!(!needs_bound_outcome(db, &terminal_bound).unwrap());
|
||||
assert_eq!(jobs::events(db).unwrap().len(), 3);
|
||||
drop(journal);
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn old_passive_configuration_cannot_enable_jobs_or_read_the_clock() {
|
||||
let config:Config=serde_json::from_value(serde_json::json!({"server_name":"example.invalid","connect_address":"127.0.0.1:443","ca_file":"C:/fixture/ca.pem","state_directory":"C:/fixture/state","enrollment_token_file":"C:/fixture/token","transport":"auto"})).unwrap();
|
||||
@@ -1319,9 +1748,13 @@ mod tests {
|
||||
assert!(!fence.allows(1, &Clock(now)));
|
||||
fence.generation.store(1, Ordering::SeqCst);
|
||||
assert!(fence.allows(1, &Clock(now)));
|
||||
assert!(!fence.allows_identity(1, true, &Clock(now)));
|
||||
fence.helper_generation.store(1, Ordering::SeqCst);
|
||||
assert!(fence.allows_identity(1, true, &Clock(now)));
|
||||
assert!(!fence.allows(2, &Clock(now)));
|
||||
assert!(!fence.allows(1, &Clock(now + chrono::Duration::seconds(1))));
|
||||
fence.close();
|
||||
assert!(!fence.allows_identity(1, true, &Clock(now)));
|
||||
assert!(!fence.allows(1, &Clock(now)));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user