Use script-only execution on Linux and preserve the enrolled helper through updates

This commit is contained in:
2026-11-18 12:01:02 +01:00
parent c692316abe
commit adc04f0148
9 changed files with 494 additions and 1010 deletions
+4 -284
View File
@@ -1,241 +1,8 @@
//! Approved artifact prefetch before a grant; Full independently admits the handoff.
//! Passive signed-release observations and retained update evidence.
use anyhow::{Context, Result, ensure};
use rmm_domain::{
agent_update::{
APPLY_CAPABILITY, ArtifactDescriptor, ArtifactRole, CHECK_CAPABILITY, Platform,
},
jobs::{Action, JobPayload},
};
use rmm_privileged_helper::{Binding, Request, control, update_admission::Accepted};
use sha2::{Digest, Sha256};
use std::{
fs::{self, File, OpenOptions},
io::{Read, Write},
os::unix::fs::{MetadataExt, OpenOptionsExt},
path::{Path, PathBuf},
time::{Duration, Instant},
};
const MAXIMUM: u64 = 256 * 1024 * 1024;
const CHUNK: u64 = 512 * 1024;
pub struct Prefetch(pub tokio::task::JoinHandle<Result<()>>);
impl Drop for Prefetch {
fn drop(&mut self) {
self.0.abort();
}
}
fn directory(path: &Path, create: bool) -> Result<()> {
if create && !path.try_exists()? {
std::fs::DirBuilder::new().mode(0o700).create(path)?;
}
let m = fs::symlink_metadata(path)?;
ensure!(
m.is_dir() && !m.file_type().is_symlink() && m.uid() == 993 && m.mode() & 0o777 == 0o700,
"UPDATE_PREFETCH_DIRECTORY"
);
Ok(())
}
use std::os::unix::fs::DirBuilderExt;
fn parent(job: &JobPayload) -> Result<PathBuf> {
ensure!(
unsafe { libc::geteuid() } == 993,
"UPDATE_PREFETCH_AGENT_UID"
);
let base = Path::new("/var/lib/rmm-agent-remote");
directory(base, false)?;
let base = base.join("update-downloads");
directory(&base, true)?;
let base = base.join(job.job_id.to_string());
directory(&base, true)?;
let base = base.join(job.attempt_id.to_string());
directory(&base, true)?;
Ok(base)
}
fn label(a: &ArtifactDescriptor) -> &'static str {
match a.role {
ArtifactRole::Agent => "agent.bin",
ArtifactRole::Helper => "helper.bin",
}
}
fn checked(path: &Path, a: &ArtifactDescriptor) -> Result<()> {
let mut f = OpenOptions::new()
.read(true)
.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
.open(path)?;
let m = f.metadata()?;
ensure!(
m.is_file()
&& m.uid() == 993
&& m.mode() & 0o777 == 0o600
&& m.nlink() == 1
&& m.len() == a.bytes,
"UPDATE_PREFETCH_FILE"
);
let mut h = Sha256::new();
let mut n = 0;
let mut buf = [0; 65536];
loop {
let size = f.read(&mut buf)?;
if size == 0 {
break;
}
n += size as u64;
ensure!(n <= a.bytes, "UPDATE_PREFETCH_LENGTH");
h.update(&buf[..size]);
}
ensure!(
n == a.bytes && format!("{:x}", h.finalize()) == a.sha256,
"UPDATE_PREFETCH_DIGEST"
);
Ok(())
}
pub async fn prefetch(client: reqwest::Client, base: String, job: JobPayload) -> Result<()> {
let Action::UpdateAgent { update } = &job.action else {
anyhow::bail!("UPDATE_ACTION_REQUIRED")
};
let manifest = update.manifest().map_err(anyhow::Error::msg)?;
ensure!(
manifest.validate() && manifest.platform == Platform::LinuxX64,
"UPDATE_PREFETCH_MANIFEST"
);
let pjob = job.clone();
let folder = tokio::task::spawn_blocking(move || parent(&pjob)).await??;
let deadline = Instant::now() + Duration::from_secs(600);
for artifact in manifest.artifacts {
ensure!(
artifact.bytes > 0 && artifact.bytes <= MAXIMUM,
"UPDATE_PREFETCH_LIMIT"
);
let path = folder.join(label(&artifact));
let existing = path.clone();
let desc = artifact.clone();
if tokio::task::spawn_blocking(move || -> Result<bool> {
if existing.try_exists()? {
checked(&existing, &desc)?;
Ok(true)
} else {
Ok(false)
}
})
.await??
{
continue;
}
let stage = folder.join(format!("{}.partial", label(&artifact)));
let stage_path = stage.clone();
let mut file = tokio::task::spawn_blocking(move || {
OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
.open(stage_path)
})
.await??;
let mut offset = 0;
let mut hash = Sha256::new();
while offset < artifact.bytes {
ensure!(
Instant::now() < deadline && chrono::Utc::now() < job.start_before,
"UPDATE_PREFETCH_EXPIRED"
);
let length = CHUNK.min(artifact.bytes - offset);
let url = format!(
"{base}/agent/v1/update-artifacts/{}?job_id={}&attempt_id={}&offset={offset}&length={length}",
artifact.artifact_id, job.job_id, job.attempt_id
);
let mut response = client.get(&url).send().await?;
let partial = offset != 0 || length != artifact.bytes;
ensure!(
response.url().as_str() == url
&& response.status()
== if partial {
reqwest::StatusCode::PARTIAL_CONTENT
} else {
reqwest::StatusCode::OK
},
"UPDATE_PREFETCH_HTTP"
);
let h = response.headers();
ensure!(
h.get("x-artifact-sha256").and_then(|v| v.to_str().ok())
== Some(artifact.sha256.as_str())
&& h.get("content-type").and_then(|v| v.to_str().ok())
== Some("application/octet-stream")
&& !h.contains_key("content-encoding")
&& response.content_length() == Some(length),
"UPDATE_PREFETCH_HEADERS"
);
if partial {
ensure!(
h.get("content-range").and_then(|v| v.to_str().ok())
== Some(
format!(
"bytes {}-{}/{}",
offset,
offset + length - 1,
artifact.bytes
)
.as_str()
),
"UPDATE_PREFETCH_RANGE"
);
} else {
ensure!(!h.contains_key("content-range"), "UPDATE_PREFETCH_RANGE");
}
let mut bytes = Vec::with_capacity(length as usize);
while let Some(chunk) = response.chunk().await? {
ensure!(
bytes.len() + chunk.len() <= length as usize,
"UPDATE_PREFETCH_BODY_LIMIT"
);
bytes.extend_from_slice(&chunk);
}
ensure!(
bytes.len() == length as usize && Instant::now() < deadline,
"UPDATE_PREFETCH_BODY_LENGTH"
);
hash.update(&bytes);
file = tokio::task::spawn_blocking(move || -> Result<File> {
file.write_all(&bytes)?;
Ok(file)
})
.await??;
offset += length;
}
ensure!(
format!("{:x}", hash.finalize()) == artifact.sha256,
"UPDATE_PREFETCH_DIGEST"
);
let dir = folder.clone();
tokio::task::spawn_blocking(move || -> Result<()> {
file.sync_all()?;
drop(file);
fs::hard_link(&stage, &path)?;
fs::remove_file(stage)?;
File::open(dir)?.sync_all()?;
checked(&path, &artifact)
})
.await??;
}
Ok(())
}
/// No Outcome is invented on acceptance; loss after write stays ambiguous.
pub fn handoff(binding: &Binding, request: &Request, grant: uuid::Uuid) -> Result<Accepted> {
let socket = std::env::var_os("RMM_FULL_MAINTENANCE_HELPER_SOCKET")
.context("UPDATE_FULL_SOCKET_REQUIRED")?;
let pin =
std::env::var_os("RMM_FULL_MAINTENANCE_POLICY_FILE").context("UPDATE_FULL_PIN_REQUIRED")?;
let pin = control::GeneralPin::read(Path::new(&pin), binding)?;
ensure!(
pin.profile == "full_maintenance_v1",
"UPDATE_FULL_PROFILE_REQUIRED"
);
control::general_probe(Path::new(&socket), binding, &pin)?;
let ack: Accepted =
control::exchange(Path::new(&socket), request, Duration::from_secs(50), 4096)?;
ack.matches(request, grant)?;
Ok(ack)
}
use rmm_domain::agent_update::Platform;
use rmm_privileged_helper::Binding;
use std::path::Path;
/// Opaque public metadata authenticated by the current protected Full peer,
/// with complete guarded history verified there. No caller JSON constructor.
@@ -342,51 +109,4 @@ impl Observer {
self.cached()?;
self.record.clone()
}
pub fn capabilities(&self) -> Vec<String> {
let Some(value) = self
.cached()
.filter(|v| v.quality == rmm_domain::agent_update::ObservationQuality::Valid)
else {
return vec![];
};
let (Some(updater), Some(predecessor)) = (&value.updater, &value.predecessor) else {
return vec![];
};
vec![
APPLY_CAPABILITY.into(),
CHECK_CAPABILITY.into(),
"update_agent".into(),
"root:update_agent".into(),
"helper.root.v1".into(),
format!("updater.binary.sha256:{}", updater.binary_sha256),
format!("updater.policy.sha256:{}", updater.policy_sha256),
format!("helper.binary.sha256:{}", predecessor.helper_sha256),
]
}
}
pub fn supported(binding: &Binding, job: &JobPayload) -> bool {
(|| -> Result<bool> {
ensure!(
unsafe { libc::geteuid() } == 993 && job.execution_identity == "root",
"UPDATE_AGENT_PRINCIPAL"
);
let Action::UpdateAgent { update } = &job.action else {
return Ok(false);
};
let observation = rmm_privileged_helper::update_admission::observe(
Path::new("/run/rmm-helper-maintenance/control.sock"),
binding,
Some(update.release_envelope.clone()),
Some(update.updater.keyring_generation),
)?;
Ok(observation.validate()
&& observation.quality == rmm_domain::agent_update::ObservationQuality::Valid
&& observation.predecessor.as_ref() == Some(&update.predecessor)
&& observation.updater.as_ref() == Some(&update.updater)
&& observation.candidate_previews.iter().any(|c| {
c.manifest_sha256 == update.manifest_sha256
&& c.peer_pin_transition == update.peer_pin_transition
}))
})()
.unwrap_or(false)
}
+4 -357
View File
@@ -1,118 +1,7 @@
//! Dedicated APT route: no fallback to generic helper, shell or SSH. Readiness
//! is nonce/root-peer/identity/profile/stack pinned before advertising or use.
use anyhow::{Context, Result, ensure};
use rmm_domain::jobs::JobPayload;
use rmm_privileged_helper::{Binding, Outcome, Request, apt as provider, readiness};
use std::{
ffi::OsString,
path::{Path, PathBuf},
};
pub fn socket_for(
job: &JobPayload,
general: Option<OsString>,
apt: Option<OsString>,
) -> Result<PathBuf> {
if job.action.full_maintenance() {
return Ok(PathBuf::from(
std::env::var_os("RMM_FULL_MAINTENANCE_HELPER_SOCKET")
.context("FULL_MAINTENANCE_SOCKET_NOT_CONFIGURED")?,
));
}
select_socket(provider::is_apt_action(job), general, apt)
}
fn select_socket(
is_apt: bool,
general: Option<OsString>,
apt: Option<OsString>,
) -> Result<PathBuf> {
if is_apt {
let socket = PathBuf::from(apt.context("APT_HELPER_SOCKET_NOT_CONFIGURED")?);
ensure!(
socket == Path::new(readiness::SOCKET),
"APT_HELPER_SOCKET_PROFILE"
);
Ok(socket)
} else {
Ok(PathBuf::from(
general.context("GENERAL_HELPER_SOCKET_NOT_CONFIGURED")?,
))
}
}
fn status(binding: &Binding) -> Result<readiness::Response> {
let path =
std::env::var_os("RMM_APT_HELPER_PROOF_FILE").context("APT_HELPER_PIN_NOT_CONFIGURED")?;
let pin = readiness::Pin::read(Path::new(&path), binding)?;
let socket =
std::env::var_os("RMM_APT_HELPER_SOCKET").context("APT_HELPER_SOCKET_NOT_CONFIGURED")?;
readiness::probe(Path::new(&socket), binding, &pin)
}
fn recovery_status(binding: &Binding) -> Result<readiness::Response> {
let path =
std::env::var_os("RMM_APT_HELPER_PROOF_FILE").context("APT_RECOVERY_PIN_REQUIRED")?;
let pin = readiness::Pin::read(Path::new(&path), binding)?;
let socket =
std::env::var_os("RMM_APT_HELPER_SOCKET").context("APT_RECOVERY_SOCKET_REQUIRED")?;
readiness::planning_for_recovery(Path::new(&socket), binding, &pin)
}
pub fn capabilities_from(response: &readiness::Response) -> Vec<String> {
if response.profile != "apt_only" {
return vec![];
}
let mut result = vec![];
if response.fixture_acceptance_ready && !response.bootstrap_recipes.is_empty() {
result.push("apt_native_acceptance".into());
}
if !response.accepted_recipes.is_empty() {
result.push("ensure_package".into());
}
if response
.accepted_recipes
.iter()
.any(|r| r == provider::UPDATE_RECIPE)
{
result.push("apply_updates".into());
}
result
}
pub fn capabilities(binding: &Binding) -> Vec<String> {
let normal = status(binding)
.map(|s| {
let mut caps = capabilities_from(&s);
if !caps.is_empty() {
let roots: Vec<String> = caps.iter().map(|c| format!("root:{c}")).collect();
caps.extend(roots);
caps.extend([
"helper.root.v1".into(),
format!("helper.policy.sha256:{}", s.provider_stack_sha256),
"apt.manifest.v2".into(),
]);
if s.fixture_acceptance_ready && !s.bootstrap_recipes.is_empty() {
caps.push("apt.native-acceptance.v1".into());
}
if !s.accepted_recipes.is_empty() {
caps.push("apt.native-accepted.v1".into());
}
}
caps
})
.unwrap_or_default();
if !normal.is_empty() {
return normal;
}
recovery_status(binding)
.map(|s| {
vec![
"apt_native_acceptance".into(),
"root:apt_native_acceptance".into(),
"helper.root.v1".into(),
format!("helper.policy.sha256:{}", s.provider_stack_sha256),
"apt.manifest.v2".into(),
"apt.native-acceptance.v1".into(),
]
})
.unwrap_or_default()
}
//! Passive APT observations for installations with a configured planning source.
use anyhow::{Context, Result};
use rmm_privileged_helper::{Binding, readiness};
use std::path::Path;
/// Called only by the periodic blocking inventory worker, never heartbeat or a
/// caller-selected live query. Failure retains honest unknown/not_configured.
@@ -142,103 +31,6 @@ pub fn planning_observation(binding: Option<&Binding>) -> serde_json::Value {
}
}
}
fn recipe_ready(job: &JobPayload, response: &readiness::Response) -> Result<()> {
ensure!(
job.execution_identity == "root" && provider::is_apt_action(job),
"APT_TYPED_ROOT_ACTION_REQUIRED"
);
let action = serde_json::to_value(&job.action)?;
let manifest: provider::AptExecutionManifest = serde_json::from_value(
action
.get("apt_manifest")
.context("APT_MANIFEST_NOT_CONFIGURED")?
.clone(),
)?;
manifest.resolved_apt_transaction.validate_fixture()?;
let recipe = &manifest.resolved_apt_transaction.recipe_id;
if let Some(reference) = &manifest.native_recovery {
let planning = response
.planning
.as_ref()
.context("APT_RECOVERY_PLANNING_REQUIRED")?;
planning.validate(chrono::Utc::now())?;
ensure!(
job.action.name() == "apt_native_acceptance"
&& manifest.schema_version == 4
&& manifest.action == "apt-native-acceptance"
&& planning.native_recovery.as_ref() == Some(reference)
&& planning.baseline_sha256 == manifest.resolved_apt_transaction.baseline_sha256
&& planning.status_fingerprint
== manifest.resolved_apt_transaction.status_fingerprint
&& planning.runtime_sha256 == manifest.runtime_sha256
&& planning.provider_sha256 == manifest.provider_sha256
&& planning.hook_sha256 == manifest.hook_sha256
&& planning.apt_config_sha256 == manifest.apt_config_sha256,
"APT_RECOVERY_FRESH_PLAN_BINDING"
);
return Ok(());
}
if job.action.name() == "apt_native_acceptance" {
ensure!(
manifest.action == "apt-native-acceptance"
&& response.fixture_acceptance_ready
&& response.bootstrap_recipes.contains(recipe),
"APT_FIXTURE_READINESS_REQUIRED"
);
} else {
ensure!(
manifest.action == "apt-execute" && response.accepted_recipes.contains(recipe),
"APT_NATIVE_ACCEPTANCE_REQUIRED"
);
ensure!(
job.action.name() != "apply_updates" || recipe == provider::UPDATE_RECIPE,
"APT_UPDATE_RECIPE_SCOPE"
);
}
Ok(())
}
pub fn supported(binding: &Binding, job: &JobPayload) -> bool {
job_status(binding, job)
.and_then(|s| recipe_ready(job, &s))
.is_ok()
}
fn job_status(binding: &Binding, job: &JobPayload) -> Result<readiness::Response> {
if matches!(&job.action,rmm_domain::jobs::Action::AptNativeAcceptance{apt_manifest,..}if apt_manifest.native_recovery.is_some())
{
recovery_status(binding)
} else {
status(binding)
}
}
pub fn invoke(binding: &Binding, job: &JobPayload, request: &Request) -> Result<Outcome> {
let socket = socket_for(
job,
std::env::var_os("RMM_HELPER_SOCKET"),
std::env::var_os("RMM_APT_HELPER_SOCKET"),
)?;
if provider::is_apt_action(job) {
recipe_ready(job, &job_status(binding, job)?)?;
}
rmm_privileged_helper::invoke(&socket, request, job.max_runtime_seconds)
}
pub fn invoke_with_control(
binding: &Binding,
job: &JobPayload,
request: &Request,
observe: impl FnMut() -> Result<()>,
) -> Result<Outcome> {
if !job.action.full_maintenance() {
return invoke(binding, job, request);
}
ensure!(
crate::helper_sync::general_supported(binding, job),
"FULL_MAINTENANCE_POLICY_REQUIRED"
);
let socket = socket_for(job, None, None)?;
rmm_privileged_helper::invoke_controlled(&socket, request, job.max_runtime_seconds, observe)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -251,149 +43,4 @@ mod tests {
assert!(observation["data"].is_null());
assert!(observation.get("received_at").is_none());
}
#[test]
fn typed_actions_require_their_own_fixture_readiness_or_accepted_recipe() {
let id = uuid::Uuid::new_v4();
let now = chrono::Utc::now();
let tx = serde_json::json!({"schema_version":1,"recipe_id":provider::INSTALL_RECIPE,"recipe_version":"1","platform":"ubuntu-24.04-amd64",
"resolver_config_version":"rmm-apt-local-v1","no_reboot":true,"status_fingerprint":"a".repeat(64),"baseline_sha256":"b".repeat(64),
"packages":[{"name":"rmm-mvp-test","architecture":"all","old_version":null,"new_version":"1.0","origin":"rmm-test",
"sha256":rmm_domain::apt_transaction::INSTALL_SHA,"size":20672,
"archive_path":format!("/var/lib/rmm-apt-transaction/archives/{}.deb",rmm_domain::apt_transaction::INSTALL_SHA)}]});
let bytes = serde_json::to_vec(&tx).unwrap();
let manifest = serde_json::json!({"type":"rmm.apt.helper-action","schema_version":2,"action":"apt-native-acceptance","execution_profile":"dev-test",
"transaction_sha256":rmm_domain::jobs::hash(&bytes),"resolved_apt_transaction":tx,"transaction_bytes":bytes,
"provider_sha256":"a".repeat(64),"hook_sha256":"a".repeat(64),"apt_config_sha256":"a".repeat(64),"runtime_sha256":"a".repeat(64),"test_result_id":"fixture-test"});
let mut job:JobPayload=serde_json::from_value(serde_json::json!({"schema_version":1,"tenant_id":id,"asset_id":id,"agent_id":id,
"enrollment_generation":1,"assignment_version":1,"job_id":id,"attempt_id":id,"execution_identity":"root",
"not_before":now,"start_before":now+chrono::Duration::minutes(1),"max_runtime_seconds":300,"max_output_bytes":4096,
"action":{"action":"apt_native_acceptance","recipe_id":provider::INSTALL_RECIPE,"recipe_version":"1","apt_manifest":manifest}})).unwrap();
assert!(job.action.validate());
let identity = readiness::Identity {
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
signing_key_id: "a".repeat(64),
};
let mut response = readiness::Response {
schema_version: 1,
challenge: id,
identity,
profile: "apt_only".into(),
provider_stack_sha256: "a".repeat(64),
fixture_acceptance_ready: true,
bootstrap_recipes: vec![provider::INSTALL_RECIPE.into()],
accepted_recipes: vec![],
ledger_device: 1,
ledger_inode: 2,
lock_device: 1,
lock_inode: 3,
planning: None,
};
recipe_ready(&job, &response).unwrap();
let mut normal = manifest;
normal["action"] = serde_json::json!("apt-execute");
job.action=serde_json::from_value(serde_json::json!({"action":"ensure_package","provider":"apt","package":"rmm-mvp-test","version":"1.0",
"architecture":"all","recipe_version":"1","installation_context":"system","recipe_id":provider::INSTALL_RECIPE,"apt_manifest":normal})).unwrap();
assert!(job.action.validate());
assert!(recipe_ready(&job, &response).is_err());
response.accepted_recipes = vec![provider::UPDATE_RECIPE.into()];
assert!(recipe_ready(&job, &response).is_err());
response.accepted_recipes = vec![provider::INSTALL_RECIPE.into()];
recipe_ready(&job, &response).unwrap();
}
#[test]
fn apt_route_cannot_fall_back_to_general_or_a_caller_path() {
assert!(select_socket(true, Some("/run/general.sock".into()), None).is_err());
assert!(
select_socket(
true,
Some("/run/general.sock".into()),
Some("/tmp/fake.sock".into())
)
.is_err()
);
assert_eq!(
select_socket(
true,
Some("/run/general.sock".into()),
Some(readiness::SOCKET.into())
)
.unwrap(),
Path::new(readiness::SOCKET)
);
}
#[test]
fn bootstrap_readiness_never_claims_package_acceptance() {
let id = readiness::Identity {
tenant_id: uuid::Uuid::new_v4(),
asset_id: uuid::Uuid::new_v4(),
agent_id: uuid::Uuid::new_v4(),
enrollment_generation: 1,
assignment_version: 1,
signing_key_id: "a".repeat(64),
};
let mut response = readiness::Response {
schema_version: 1,
challenge: uuid::Uuid::new_v4(),
identity: id,
profile: "apt_only".into(),
provider_stack_sha256: "a".repeat(64),
fixture_acceptance_ready: true,
bootstrap_recipes: vec![provider::INSTALL_RECIPE.into()],
accepted_recipes: vec![],
ledger_device: 1,
ledger_inode: 2,
lock_device: 1,
lock_inode: 3,
planning: None,
};
assert_eq!(capabilities_from(&response), vec!["apt_native_acceptance"]);
response.accepted_recipes = vec![provider::INSTALL_RECIPE.into()];
assert!(!capabilities_from(&response).contains(&"apply_updates".into()));
response
.accepted_recipes
.push(provider::UPDATE_RECIPE.into());
assert!(capabilities_from(&response).contains(&"apply_updates".into()));
}
#[test]
fn generic_script_never_uses_apt_socket() {
let now = chrono::Utc::now();
let id = uuid::Uuid::new_v4();
let job = JobPayload {
schema_version: 1,
tenant_id: id,
asset_id: id,
agent_id: id,
enrollment_generation: 1,
assignment_version: 1,
job_id: id,
attempt_id: id,
action: rmm_domain::jobs::Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "true".into(),
arguments: vec![],
expected_stdout: None,
windows_execution_profile: None,
privileged_script_profile: None,
},
execution_identity: "root".into(),
not_before: now,
start_before: now + chrono::Duration::minutes(1),
max_runtime_seconds: 300,
max_output_bytes: 4096,
};
assert!(socket_for(&job, None, Some(readiness::SOCKET.into())).is_err());
assert_eq!(
socket_for(
&job,
Some("/run/general.sock".into()),
Some(readiness::SOCKET.into())
)
.unwrap(),
Path::new("/run/general.sock")
);
}
}
+61 -90
View File
@@ -41,16 +41,12 @@ fn map(s: &root::Snapshot) -> reconcile::HelperSnapshot {
.collect(),
}
}
fn general(binding: &Binding, full: bool) -> Result<control::Response> {
let (socket_name, pin_name, profile) = if full {
(
"RMM_FULL_MAINTENANCE_HELPER_SOCKET",
"RMM_FULL_MAINTENANCE_POLICY_FILE",
"full_maintenance_v1",
)
} else {
("RMM_HELPER_SOCKET", "RMM_HELPER_POLICY_FILE", "general")
};
fn script_helper(binding: &Binding) -> Result<control::Response> {
let (socket_name, pin_name, profile) = (
"RMM_FULL_MAINTENANCE_HELPER_SOCKET",
"RMM_FULL_MAINTENANCE_POLICY_FILE",
"full_maintenance_v1",
);
let socket = std::env::var_os(socket_name).context("NO_GENERAL_HELPER")?;
let path = std::env::var_os(pin_name).context("GENERAL_POLICY_PIN_REQUIRED")?;
let pin = control::GeneralPin::read(Path::new(&path), binding)?;
@@ -207,97 +203,72 @@ impl Helpers {
Ok(applied.as_ref().map(map))
}
}
pub fn general_capabilities(binding: &Binding) -> Vec<String> {
let mut caps = profile_capabilities(binding, false);
caps.extend(profile_capabilities(binding, true));
caps.sort();
caps.dedup();
caps
}
fn profile_capabilities(binding: &Binding, full: bool) -> Vec<String> {
pub fn script_capabilities(binding: &Binding) -> Vec<String> {
(|| -> Result<Vec<String>> {
let response = general(binding, full)?;
let policy = response.policy.context("NO_GENERAL_POLICY")?;
let response = script_helper(binding)?;
let binary = response
.helper_binary_sha256
.context("SCRIPT_HELPER_BINARY_PROOF_REQUIRED")?;
ensure!(
rmm_agent_linux::actions::Interpreter::PosixSh.available(),
"SCRIPT_INTERPRETER_UNAVAILABLE"
);
let mut caps = vec![
"run_script".into(),
"root:run_script".into(),
"helper.root.v1".into(),
rmm_domain::jobs::FULL_MAINTENANCE_CAPABILITY.into(),
format!("helper.policy.sha256:{}", response.policy_sha256),
format!("helper.binary.sha256:{binary}"),
];
if full {
ensure!(
policy.schema_version == 2
&& policy.profile == "full_maintenance_v1"
&& policy.allowed_script_profiles.as_deref()
== Some(&[rmm_domain::jobs::PrivilegedScriptProfile::FullMaintenanceV1]),
"FULL_MAINTENANCE_POLICY_REQUIRED"
);
let binary = policy
.helper_binary_sha256
.as_ref()
.context("FULL_MAINTENANCE_BINARY_PIN_REQUIRED")?;
ensure!(
response.helper_binary_sha256.as_ref() == Some(binary),
"FULL_MAINTENANCE_BINARY_PROOF_REQUIRED"
);
caps.push(format!("helper.binary.sha256:{binary}"));
caps.push(rmm_domain::jobs::FULL_MAINTENANCE_CAPABILITY.into());
}
let mut actions = vec![];
if policy.allow_root_scripts && rmm_agent_linux::actions::Interpreter::PosixSh.available() {
actions.push("run_script");
}
if !policy.allowed_services.is_empty() {
actions.extend(["set_service_state", "query_logs"]);
}
if policy.allow_reboot {
actions.push("reboot");
}
for action in actions {
caps.push(action.into());
caps.push(format!("root:{action}"));
if let Some(release) = std::env::var("RMM_SCRIPT_HELPER_RELEASE")
.ok()
.and_then(|value| Uuid::parse_str(&value).ok())
.filter(|id| !id.is_nil())
{
caps.push(format!("script.helper.release:{release}"));
}
Ok(caps)
})()
.unwrap_or_default()
}
pub fn general_supported(binding: &Binding, job: &rmm_domain::jobs::JobPayload) -> bool {
(|| -> Result<bool> {
let p = general(binding, job.action.full_maintenance())?
.policy
.context("NO_GENERAL_POLICY")?;
Ok(match &job.action {
rmm_domain::jobs::Action::RunScript { interpreter, .. } => {
p.allow_root_scripts
&& job.action.valid_execution_identity(&job.execution_identity)
&& job
.action
.valid_job_limits(job.max_runtime_seconds, job.max_output_bytes)
&& (!job.action.full_maintenance()
|| p.allowed_script_profiles.as_ref().is_some_and(|v| {
v.contains(
&rmm_domain::jobs::PrivilegedScriptProfile::FullMaintenanceV1,
)
}))
&& match interpreter.as_str() {
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
_ => false,
}
}
rmm_domain::jobs::Action::SetServiceState { unit, state } => {
p.allowed_services.contains(unit)
&& matches!(state.as_str(), "running" | "stopped")
&& job.max_runtime_seconds >= 180
}
rmm_domain::jobs::Action::QueryLogs { unit, .. } => {
p.allowed_services.contains(unit) && job.max_runtime_seconds >= 20
}
rmm_domain::jobs::Action::Reboot { .. } => {
p.allow_reboot && job.max_runtime_seconds >= 60
}
pub fn script_supported(binding: &Binding, job: &rmm_domain::jobs::JobPayload) -> bool {
let rmm_domain::jobs::Action::RunScript { interpreter, .. } = &job.action else {
return false;
};
job.execution_identity == "root"
&& job.action.full_maintenance()
&& job.action.valid_execution_identity(&job.execution_identity)
&& job
.action
.valid_job_limits(job.max_runtime_seconds, job.max_output_bytes)
&& match interpreter.as_str() {
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
_ => false,
})
})()
.unwrap_or(false)
}
&& script_helper(binding).is_ok()
}
pub fn invoke_script(
binding: &Binding,
job: &rmm_domain::jobs::JobPayload,
request: &rmm_privileged_helper::Request,
observe: impl FnMut() -> Result<()>,
) -> Result<rmm_privileged_helper::Outcome> {
ensure!(
script_supported(binding, job),
"SCRIPT_HELPER_POLICY_REQUIRED"
);
let socket = std::env::var_os("RMM_FULL_MAINTENANCE_HELPER_SOCKET")
.context("SCRIPT_HELPER_SOCKET_REQUIRED")?;
rmm_privileged_helper::invoke_controlled(
Path::new(&socket),
request,
job.max_runtime_seconds,
observe,
)
}
impl Helpers {
+45 -225
View File
@@ -1,183 +1,46 @@
use anyhow::{Result, ensure};
use anyhow::Result;
#[cfg(test)]
use prost::Message;
#[cfg(test)]
use rmm_domain::jobs::hash;
use rmm_domain::jobs::{Action, JobPayload};
use rmm_privileged_helper::{Binding, Outcome};
use rmm_privileged_helper::Binding;
use rmm_protocol::*;
use rusqlite::Connection;
#[cfg(test)]
use uuid::Uuid;
pub fn supported(job: &JobPayload) -> bool {
if rmm_privileged_helper::apt::is_apt_action(job) {
// A configured environment path alone is never a capability proof.
// Caller binding is independently provisioned, not taken from the job.
pub fn supported_with_binding(job: &JobPayload, binding: Option<&Binding>) -> bool {
let Action::RunScript { interpreter, .. } = &job.action else {
return false;
};
if unsafe { libc::geteuid() } == 0
|| !job.action.valid_execution_identity(&job.execution_identity)
{
return false;
}
// The network process must never execute a script with accidental root.
supported_as(
job,
unsafe { libc::geteuid() },
std::env::var_os("RMM_HELPER_SOCKET").is_some(),
)
}
pub fn supported_with_binding(job: &JobPayload, binding: Option<&Binding>) -> bool {
if matches!(job.action, Action::UpdateAgent { .. }) {
return binding.is_some_and(|b| crate::agent_update_apply::supported(b, job));
}
if rmm_privileged_helper::apt::is_apt_action(job) {
// SAFETY: geteuid has no preconditions.
return unsafe { libc::geteuid() } != 0
&& binding.is_some_and(|b| crate::apt::supported(b, job));
}
if job.execution_identity == "root" {
return unsafe { libc::geteuid() } != 0
&& binding.is_some_and(|b| crate::helper_sync::general_supported(b, job));
}
supported(job)
}
fn supported_as(job: &JobPayload, uid: u32, helper_configured: bool) -> bool {
let unprivileged = uid != 0;
if job.execution_identity == "root" && unprivileged {
return helper_configured
&& matches!(
job.action,
Action::RunScript { .. } | Action::SetServiceState { .. } | Action::Reboot { .. }
);
}
job.execution_identity == "agent_service"
&& unprivileged
&& match &job.action {
Action::RunScript { interpreter, .. } => match interpreter.as_str() {
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
_ => false,
},
Action::RefreshInventory { scope } => scope == "linux.base",
Action::QueryLogs { .. } => log_request(job).is_ok(),
_ => false,
match job.execution_identity.as_str() {
"root" => {
job.action.full_maintenance()
&& binding.is_some_and(|b| crate::helper_sync::script_supported(b, job))
}
"agent_service" => match interpreter.as_str() {
"posix_sh" => rmm_agent_linux::actions::Interpreter::PosixSh.available(),
"bash" => rmm_agent_linux::actions::Interpreter::Bash.available(),
_ => false,
},
_ => false,
}
}
pub use rmm_agent_core::jobs::{
denied_request, ack, event, events, interrupted, interrupted_request, next, no_start, permit_request,
recover_interrupted_outcome, requesting, reserve_request, result, transition,
ack, denied_request, event, events, interrupted, interrupted_request, next, no_start,
permit_request, recover_interrupted_outcome, requesting, reserve_request, result, transition,
};
pub fn accept(db: &Connection, binding: &Binding, offer: &JobOffer) -> Result<bool> {
rmm_agent_core::jobs::accept(db, binding, offer, |job| {
supported_with_binding(job, Some(binding))
})
}
pub fn execute(job: &JobPayload) -> Outcome {
let result = match &job.action {
Action::RunScript { .. } => rmm_privileged_helper::script(job),
Action::RefreshInventory { .. } => rmm_agent_linux::collect().and_then(|i| {
let bytes = serde_json::to_vec(&i)?;
ensure!(bytes.len() <= job.max_output_bytes as usize, "INVENTORY_OUTPUT_LIMIT");
Ok(Outcome { state: "succeeded".into(), stdout: bytes, stderr: vec![], exit_code: None,
output_truncated: false, reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":true,"source_id":"linux.base","observed_at":chrono::Utc::now(),"completeness":"partial"}), error_code: String::new() })
}),
Action::QueryLogs { .. } => query_logs(job),
_ => Err(anyhow::anyhow!("unsupported action")),
};
result.unwrap_or_else(|_| Outcome::unknown("EXECUTOR_EFFECT_UNKNOWN"))
}
fn allowed_log_units() -> Result<Vec<rmm_agent_linux::actions::ServiceUnit>> {
let path = std::env::var_os("RMM_LOG_UNITS_FILE")
.ok_or_else(|| anyhow::anyhow!("LOG_ALLOWLIST_NOT_CONFIGURED"))?;
let path = std::path::Path::new(&path);
rmm_privileged_helper::private_path(path, unsafe { libc::geteuid() }, false)?;
ensure!(
std::fs::metadata(path)?.len() <= 32768,
"LOG_ALLOWLIST_SIZE"
);
let units: Vec<String> = serde_json::from_slice(&std::fs::read(path)?)?;
ensure!(
!units.is_empty() && units.len() <= 128,
"LOG_ALLOWLIST_SIZE"
);
units
.iter()
.map(|u| rmm_agent_linux::actions::ServiceUnit::new(u))
.collect()
}
pub fn logs_available() -> bool {
allowed_log_units().is_ok()
&& rmm_agent_linux::inventory::capabilities()
.get("logs.journald")
.is_some_and(|o| o.data == Some(true))
}
fn log_request(job: &JobPayload) -> Result<rmm_agent_linux::inventory::JournalRequest> {
log_request_for(job, &allowed_log_units()?)
}
fn log_request_for(
job: &JobPayload,
allowed: &[rmm_agent_linux::actions::ServiceUnit],
) -> Result<rmm_agent_linux::inventory::JournalRequest> {
let Action::QueryLogs {
unit,
since,
until,
max_lines,
} = &job.action
else {
anyhow::bail!("NOT_LOG_QUERY")
};
ensure!(job.max_runtime_seconds >= 20, "LOG_RUNTIME_TOO_SHORT");
rmm_agent_linux::inventory::JournalRequest::new(
rmm_agent_linux::actions::ServiceUnit::new(unit)?,
allowed,
u64::try_from(since.timestamp())?,
u64::try_from(until.timestamp())?,
*max_lines as usize,
(job.max_output_bytes as usize).min(256 * 1024),
)?
.precise_window(*since, *until)
}
fn query_logs(job: &JobPayload) -> Result<Outcome> {
let observation = rmm_agent_linux::inventory::query_journal(&log_request(job)?);
let valid = observation.quality == rmm_agent_linux::inventory::Quality::Valid;
let error = observation.error_code.clone().unwrap_or_default();
let mut value = serde_json::to_value(&observation)?;
while serde_json::to_vec(&value)?.len() > job.max_output_bytes as usize {
let Some(lines) = value["data"]["json_lines"]
.as_array_mut()
.filter(|v| !v.is_empty())
else {
return Ok(Outcome {
state: "failed".into(),
..Outcome::unknown("LOG_RESULT_METADATA_EXCEEDS_OUTPUT_LIMIT")
});
};
lines.remove(0);
value["truncated"] = serde_json::json!(true);
value["completeness"] = serde_json::json!("partial");
value["error_code"] = serde_json::json!("LOG_SERIALIZED_OUTPUT_LIMIT");
}
Ok(Outcome {
state: if valid { "succeeded" } else { "failed" }.into(),
stdout: serde_json::to_vec(&value)?,
stderr: vec![],
exit_code: None,
output_truncated: value["truncated"] == true,
reboot_required: false,
evidence: serde_json::json!({"postconditions_passed":valid,"source_id":observation.source_id,
"observed_at":value["observed_at"],"quality":value["quality"],"completeness":value["completeness"],
"cursor_status":"not_advanced_no_ack_contract","provider_error_code":error}),
error_code: if valid {
String::new()
} else if error.is_empty() {
"LOG_PROVIDER_UNAVAILABLE".into()
} else {
error
},
})
}
/// With no backend SyncResponse contract, a reconnect/restart cannot prove an
/// earlier reservation never started. Telemetry and evidence replay continue;
/// all new effects stop. Do not paper over this with a new start request.
#[cfg(test)]
pub fn begin_session(db: &Connection) -> Result<bool> {
@@ -228,8 +91,13 @@ mod tests {
assignment_version: 2,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RefreshInventory {
scope: "linux.base".into(),
action: Action::RunScript {
interpreter: "posix_sh".into(),
script_utf8: "true".into(),
arguments: vec![],
expected_stdout: None,
windows_execution_profile: None,
privileged_script_profile: None,
},
execution_identity: "agent_service".into(),
not_before: now,
@@ -254,7 +122,7 @@ mod tests {
let mut db = storage::open(&path).unwrap();
let job = binding.job(&offer.payload_bytes).unwrap();
if unsafe { libc::geteuid() } == 0 {
assert!(!supported(&job));
assert!(!supported_with_binding(&job, Some(&binding)));
assert!(accept(&db, &binding, &offer).is_err());
} else {
assert!(accept(&db, &binding, &offer).unwrap());
@@ -326,20 +194,24 @@ mod tests {
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn network_process_never_executes_as_accidental_root() {
fn legacy_actions_and_unbound_root_are_rejected_before_any_reservation() {
let (path, binding, offer) = fixture();
let mut job = binding.job(&offer.payload_bytes).unwrap();
assert!(!supported_as(&job, 0, false));
assert!(!supported_as(&job, 0, true));
job.execution_identity = "root".into();
assert!(!supported_as(&job, 1000, false));
assert!(!supported_as(&job, 0, true));
job.action = Action::Reboot {
reason: "explicit root only".into(),
reconnect_timeout_seconds: 60,
dependency_group: "test".into(),
};
assert!(supported_as(&job, 1000, true));
assert!(!supported_with_binding(&job, Some(&binding)));
for identity in ["agent_service", "root"] {
job.execution_identity = identity.into();
job.action = Action::Reboot {
reason: "legacy fixture".into(),
reconnect_timeout_seconds: 60,
dependency_group: "test".into(),
};
assert!(!supported_with_binding(&job, Some(&binding)));
job.action = Action::RefreshInventory {
scope: "linux.base".into(),
};
assert!(!supported_with_binding(&job, Some(&binding)));
}
std::fs::remove_dir_all(path).unwrap();
}
#[test]
@@ -363,56 +235,4 @@ mod tests {
drop(db);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn live_logs_enforce_actual_action_window_allowlist_limits_and_runtime() {
let (path, binding, offer) = fixture();
let mut job = binding.job(&offer.payload_bytes).unwrap();
let unit = rmm_agent_linux::actions::ServiceUnit::new("rmm-test.service").unwrap();
let now = chrono::Utc::now();
job.max_runtime_seconds = 20;
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::minutes(1),
until: now,
max_lines: 10,
};
assert!(log_request_for(&job, &[]).is_err());
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_ok());
job.max_runtime_seconds = 10;
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
job.max_runtime_seconds = 20;
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::hours(25),
until: now,
max_lines: 10,
};
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
job.action = Action::QueryLogs {
unit: unit.as_str().into(),
since: now - chrono::Duration::minutes(1),
until: now,
max_lines: 1001,
};
assert!(log_request_for(&job, std::slice::from_ref(&unit)).is_err());
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn refresh_is_a_new_measurement_with_real_observation_time() {
let (path, binding, offer) = fixture();
let job = binding.job(&offer.payload_bytes).unwrap();
let before = chrono::Utc::now();
let outcome = execute(&job);
assert_eq!(outcome.state, "succeeded");
let observed =
chrono::DateTime::parse_from_rfc3339(outcome.evidence["observed_at"].as_str().unwrap())
.unwrap();
assert!(observed >= before);
assert!(
serde_json::from_slice::<rmm_domain::Inventory>(&outcome.stdout)
.unwrap()
.validate()
);
std::fs::remove_dir_all(path).unwrap();
}
}
+5 -38
View File
@@ -425,15 +425,8 @@ impl<C: reconcile::SyncClock> Agent<C> {
|| binding.verify(&request, Utc::now()).is_err()
{
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
} else if matches!(job.action, rmm_domain::jobs::Action::UpdateAgent { .. }) {
match agent_update_apply::handoff(&binding, &request, grant.grant_id) {
// Full now owns this original. The stable updater may
// stop us. Reconciliation alone supplies its final result.
Ok(_) => return,
Err(_) => rmm_privileged_helper::Outcome::unknown("UPDATE_HANDOFF_UNCONFIRMED"),
}
} else if job.execution_identity == "root" {
apt::invoke_with_control(&binding, &job, &request, || {
helper_sync::invoke_script(&binding, &job, &request, || {
ensure!(
!cancellation.load(Ordering::SeqCst)
&& fence.running_generation.load(Ordering::SeqCst) == generation
@@ -446,7 +439,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
rmm_privileged_helper::Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN")
})
} else {
jobs::execute(&job)
rmm_privileged_helper::script(&job).unwrap_or_else(|_| rmm_privileged_helper::Outcome::unknown("EXECUTOR_EFFECT_UNKNOWN"))
};
let _ = completed.blocking_send((offer, grant, outcome));
});
@@ -481,15 +474,7 @@ impl<C: reconcile::SyncClock> Agent<C> {
if rmm_agent_linux::actions::Interpreter::PosixSh.available() {
capabilities.push("run_script".into());
}
capabilities.push("refresh_inventory".into());
if jobs::logs_available() {
capabilities.push("query_logs".into());
}
capabilities.extend(apt::capabilities(binding));
capabilities.extend(helper_sync::general_capabilities(binding));
if self.clock.trusted() {
capabilities.extend(self.update_observer.capabilities());
}
capabilities.extend(helper_sync::script_capabilities(binding));
}
capabilities.sort();
capabilities.dedup();
@@ -670,8 +655,6 @@ impl<C: reconcile::SyncClock> Agent<C> {
let mut first_heartbeat = true;
let mut health_task: Option<PendingUpdateHealth> = None;
let mut health_inventory_prefix: Option<String> = None;
let mut update_prefetch: Option<(JobOffer, agent_update_apply::Prefetch)> = None;
let mut prefetched = std::collections::HashSet::new();
loop {
tokio::select! {
Some(result) = self.inventory_rx.recv() => {
@@ -748,7 +731,6 @@ impl<C: reconcile::SyncClock> Agent<C> {
// First preserve the original completion without unlocking.
// A startup can replay it if the journal transaction is lost.
let state = outcome.state.clone();
let refresh_capabilities=state=="succeeded" && serde_json::from_slice::<rmm_domain::jobs::JobPayload>(&offer.payload_bytes)?.action.name()=="apt_native_acceptance";
rmm_privileged_helper::persist_outcome(&self.ledger,&grant,&outcome)?;
let serialized = serde_json::to_vec(&outcome)?;
self.record(generation, &offer, &grant.grant_id.to_string(), outcome)?;
@@ -756,17 +738,9 @@ impl<C: reconcile::SyncClock> Agent<C> {
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
if state == "unknown" { eprintln!("job effect unknown: device execution lock retained"); }
needs_sync=true;self.replay(&ws, generation).await?;
if refresh_capabilities {return Ok(());}
}
_ = tick.tick() => {
self.replay(&ws, generation).await?;
if update_prefetch.as_ref().is_some_and(|(_,task)|task.0.is_finished()) {
let (offer, mut task)=update_prefetch.take().context("UPDATE_PREFETCH_REQUIRED")?;
match (&mut task.0).await {
Ok(Ok(())) => {prefetched.insert((offer.job_id.clone(),offer.attempt_id.clone()));},
_ => {self.record(generation,&offer,"",jobs::no_start("failed","UPDATE_ARTIFACT_PREFETCH_FAILED"))?;needs_sync=true;}
}
}
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
let update=self.update_proof()?;
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
@@ -802,18 +776,11 @@ impl<C: reconcile::SyncClock> Agent<C> {
let job = self.binding.as_ref().context("binding")?.job(&offer.payload_bytes)?;
if Utc::now() >= job.start_before {
self.record(generation, &offer, "", jobs::no_start("expired", "START_DEADLINE_EXPIRED"))?;
} else if !matches!(job.action,rmm_domain::jobs::Action::RunScript{..}) {
self.record(generation, &offer, "", jobs::no_start("failed","LEGACY_ACTION_RETIRED"))?;
} else if !self.ready(generation,job.execution_identity=="root") {
needs_sync=true;
} else if Utc::now() >= job.not_before {
if matches!(job.action,rmm_domain::jobs::Action::UpdateAgent{..}) && !prefetched.contains(&(offer.job_id.clone(),offer.attempt_id.clone())) {
if update_prefetch.is_none() {
let identity=self.certificates.current(Utc::now().timestamp())?;
let client=https_channel::mtls_client(self.address,&self.ca,&identity.certificate_pem,&identity.private_key_pem)?;
let base=format!("https://{}:{}",https_channel::server_name()?,self.address.port());
update_prefetch=Some((offer.clone(),agent_update_apply::Prefetch(tokio::spawn(agent_update_apply::prefetch(client,base,job)))));
}
continue;
}
// Reserve before send. A lost response must never cause
// a second permit request for this attempt.
// The root updater holds this lock exclusively while it checks