1037 lines
46 KiB
Rust
1037 lines
46 KiB
Rust
mod agent_update_catalog;
|
|
mod agent_update_health;
|
|
mod apt;
|
|
mod bootstrap;
|
|
mod certificate_rotation;
|
|
mod helper_sync;
|
|
mod https_channel;
|
|
mod jobs;
|
|
mod reconcile;
|
|
#[cfg(test)]
|
|
mod runtime_tests;
|
|
mod storage;
|
|
mod transport;
|
|
mod wire_channel;
|
|
|
|
use anyhow::{Context, Result, ensure};
|
|
use chrono::Utc;
|
|
use openssl::{
|
|
ec::{EcGroup, EcKey},
|
|
hash::MessageDigest,
|
|
nid::Nid,
|
|
pkey::PKey,
|
|
x509::{X509NameBuilder, X509Req},
|
|
};
|
|
use prost::Message;
|
|
use rmm_protocol::*;
|
|
use sha2::{Digest, Sha256};
|
|
use std::{
|
|
env, fs,
|
|
path::{Path, PathBuf},
|
|
sync::{
|
|
Arc,
|
|
atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
|
|
},
|
|
time::{Duration, Instant},
|
|
};
|
|
use uuid::Uuid;
|
|
|
|
struct Agent<C: reconcile::SyncClock = reconcile::NativeClock> {
|
|
address: std::net::SocketAddr,
|
|
ca: Vec<u8>,
|
|
id: String,
|
|
boot: String,
|
|
instance: String,
|
|
db: rusqlite::Connection,
|
|
started: Instant,
|
|
clock: rmm_privileged_helper::TrustedClock,
|
|
binding: Option<rmm_privileged_helper::Binding>,
|
|
ledger: rusqlite::Connection,
|
|
completed_tx: tokio::sync::mpsc::Sender<Completion>,
|
|
completed_rx: tokio::sync::mpsc::Receiver<Completion>,
|
|
inventory_tx: tokio::sync::mpsc::Sender<Result<(Vec<u8>, i64)>>,
|
|
inventory_rx: tokio::sync::mpsc::Receiver<Result<(Vec<u8>, i64)>>,
|
|
inventory_inflight: bool,
|
|
running_cancels: std::collections::HashMap<(String, String), Arc<AtomicBool>>,
|
|
sync: Option<reconcile::ClientSync>,
|
|
sync_clock: C,
|
|
helpers: Option<helper_sync::Helpers>,
|
|
prepared: Option<reconcile::PreparedSync>,
|
|
fence: Arc<EffectFence>,
|
|
mode: wire_channel::Mode,
|
|
certificates: certificate_rotation::RotationStore,
|
|
rotation_policy: certificate_rotation::RotationPolicy,
|
|
catalog: agent_update_catalog::Checker,
|
|
}
|
|
#[derive(Default)]
|
|
struct EffectFence {
|
|
running_generation: AtomicU64,
|
|
generation: AtomicU64,
|
|
deadline: AtomicI64,
|
|
root: AtomicBool,
|
|
}
|
|
fn report_closed_gate(error: &anyhow::Error) {
|
|
let text = error.to_string();
|
|
let code = if text.len() <= 96
|
|
&& text
|
|
.bytes()
|
|
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
|
|
{
|
|
text.as_str()
|
|
} else {
|
|
"SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE"
|
|
};
|
|
eprintln!("agent effect gate closed: {code}");
|
|
}
|
|
impl EffectFence {
|
|
fn close(&self) {
|
|
self.running_generation.store(0, Ordering::SeqCst);
|
|
self.close_starts();
|
|
}
|
|
// A signed Sync/start lease authorizes NEW starts. Its natural expiry
|
|
// cannot retroactively stop an approved bounded process (spec22.3).
|
|
// Running effects instead observe their exact cancellation and current
|
|
// connection generation; transport/signed refusal closes both fences.
|
|
fn close_starts(&self) {
|
|
self.generation.store(0, Ordering::SeqCst);
|
|
self.root.store(false, Ordering::SeqCst);
|
|
}
|
|
fn allows(&self, generation: u64, root: bool) -> bool {
|
|
self.generation.load(Ordering::SeqCst) == generation
|
|
&& generation > 0
|
|
&& Utc::now().timestamp_millis() < self.deadline.load(Ordering::SeqCst)
|
|
&& (!root || self.root.load(Ordering::SeqCst))
|
|
}
|
|
}
|
|
struct PendingUpdateHealth(tokio::task::JoinHandle<Result<()>>);
|
|
impl Drop for PendingUpdateHealth {
|
|
fn drop(&mut self) {
|
|
self.0.abort();
|
|
}
|
|
}
|
|
type Completion = (
|
|
JobOffer,
|
|
rmm_domain::jobs::ExecutionPermitPayload,
|
|
rmm_privileged_helper::Outcome,
|
|
);
|
|
impl<C: reconcile::SyncClock> Agent<C> {
|
|
fn close_gate(&mut self) {
|
|
self.fence.close();
|
|
if let Some(sync) = &mut self.sync {
|
|
sync.close();
|
|
}
|
|
self.prepared = None;
|
|
}
|
|
fn channel(&self, generation: u64) -> Result<reconcile::Channel> {
|
|
Ok(reconcile::Channel {
|
|
connection_generation: generation,
|
|
boot_id: Uuid::parse_str(&self.boot)?,
|
|
})
|
|
}
|
|
fn ready(&self, generation: u64, root: bool) -> bool {
|
|
self.channel(generation).is_ok_and(|c| {
|
|
self.sync
|
|
.as_ref()
|
|
.is_some_and(|s| s.can_start(&c, root, &self.sync_clock))
|
|
})
|
|
}
|
|
async fn start_sync(&mut self, wire: &wire_channel::Handle, generation: u64) -> Result<()> {
|
|
self.fence.close_starts();
|
|
let helpers = self.helpers.clone();
|
|
let snapshot = tokio::task::spawn_blocking(move || match helpers {
|
|
Some(h) => h.snapshot(generation),
|
|
None => Ok(None),
|
|
})
|
|
.await??;
|
|
let channel = self.channel(generation)?;
|
|
let Some(sync) = &mut self.sync else {
|
|
return Ok(());
|
|
};
|
|
let prepared = sync.prepare(
|
|
&mut self.db,
|
|
&self.ledger,
|
|
channel,
|
|
snapshot.as_ref(),
|
|
&self.sync_clock,
|
|
)?;
|
|
wire.send(&prepared.envelope).await?;
|
|
self.prepared = Some(prepared);
|
|
Ok(())
|
|
}
|
|
async fn apply_sync(&mut self, generation: u64, response: SyncResponse) -> Result<()> {
|
|
let prepared = self.prepared.take().context("UNSOLICITED_SYNC_RESPONSE")?;
|
|
let sync = self.sync.as_mut().context("SYNC_BINDING_REQUIRED")?;
|
|
let applied = sync.apply(&mut self.db, &response, &self.sync_clock)?;
|
|
sync.apply_consumption(&mut self.ledger, &self.sync_clock)?;
|
|
let reason = if applied
|
|
.reason
|
|
.bytes()
|
|
.all(|b| b.is_ascii_uppercase() || b == b'_')
|
|
{
|
|
applied.reason.as_str()
|
|
} else {
|
|
"RECONCILIATION_REQUIRED"
|
|
};
|
|
eprintln!(
|
|
"agent reconciliation committed: decision={}, receipts={}, reason={}",
|
|
applied.decision_id,
|
|
applied.acknowledged_message_ids.len(),
|
|
reason
|
|
);
|
|
if let Some(instruction) = applied.helper_instruction {
|
|
let helpers = self
|
|
.helpers
|
|
.clone()
|
|
.context("HELPER_SYNC_CLIENT_REQUIRED")?;
|
|
let request = prepared.request.clone();
|
|
let snapshot =
|
|
tokio::task::spawn_blocking(move || helpers.apply(&instruction.response, &request))
|
|
.await??
|
|
.context("HELPER_SYNC_ACK_REQUIRED")?;
|
|
if self.ready(generation, false) {
|
|
self.sync
|
|
.as_mut()
|
|
.context("SYNC_REQUIRED")?
|
|
.confirm_helper(&snapshot)?;
|
|
} else if snapshot
|
|
.attempts
|
|
.iter()
|
|
.any(|a| !a.update_handoff_bytes.is_empty())
|
|
{
|
|
self.sync
|
|
.as_mut()
|
|
.context("SYNC_REQUIRED")?
|
|
.confirm_update_helper(&snapshot)?;
|
|
}
|
|
}
|
|
if self.ready(generation, false) {
|
|
let deadline = self
|
|
.sync
|
|
.as_ref()
|
|
.and_then(|s| s.validity_deadline())
|
|
.context("SIGNED_LEASE_REQUIRED")?;
|
|
self.fence
|
|
.running_generation
|
|
.store(generation, Ordering::SeqCst);
|
|
self.fence
|
|
.deadline
|
|
.store(deadline.timestamp_millis(), Ordering::SeqCst);
|
|
self.fence
|
|
.root
|
|
.store(self.ready(generation, true), Ordering::SeqCst);
|
|
self.fence.generation.store(generation, Ordering::SeqCst);
|
|
} else {
|
|
// A valid signed refusal is a fence too, not just parse/I/O errors.
|
|
self.fence.close();
|
|
}
|
|
Ok(())
|
|
}
|
|
fn update_proof(&self) -> Result<Option<rmm_domain::agent_update::UpdateInflightProof>> {
|
|
self.binding
|
|
.as_ref()
|
|
.map(|binding| {
|
|
rmm_agent_core::update_health_inventory::current_proof(&self.db, binding)
|
|
})
|
|
.transpose()
|
|
.map(Option::flatten)
|
|
}
|
|
fn start_update_health(
|
|
&self,
|
|
proof: &rmm_domain::agent_update::UpdateInflightProof,
|
|
) -> Result<Option<PendingUpdateHealth>> {
|
|
if !self.clock.trusted() {
|
|
return Ok(None);
|
|
}
|
|
let binding = self.binding.as_ref().context("UPDATE_BINDING_REQUIRED")?;
|
|
let Some(context) =
|
|
agent_update_health::context(&self.db, binding, proof, self.sync_clock.now()?)?
|
|
else {
|
|
return Ok(None);
|
|
};
|
|
let header = context.handoff().clone();
|
|
let key = context.health_public_key();
|
|
let identity = self.certificates.current(Utc::now().timestamp())?;
|
|
let client = https_channel::mtls_client(
|
|
self.address,
|
|
&self.ca,
|
|
&identity.certificate_pem,
|
|
&identity.private_key_pem,
|
|
)?;
|
|
let helpers = self
|
|
.helpers
|
|
.clone()
|
|
.context("UPDATE_FULL_HELPER_REQUIRED")?;
|
|
let address = self.address;
|
|
Ok(Some(PendingUpdateHealth(tokio::spawn(async move {
|
|
let receipt = agent_update_health::request(&client, address, &context).await?;
|
|
tokio::task::spawn_blocking(move || {
|
|
helpers.submit_update_health(&header, key, &receipt)
|
|
})
|
|
.await??;
|
|
Ok(())
|
|
}))))
|
|
}
|
|
fn record(
|
|
&mut self,
|
|
generation: u64,
|
|
offer: &JobOffer,
|
|
grant: &str,
|
|
outcome: rmm_privileged_helper::Outcome,
|
|
) -> Result<()> {
|
|
let mut envelope = self.envelope(generation, Payload::JobResult(JobResult::default()))?;
|
|
let tx = self.db.transaction()?;
|
|
let revision = jobs::transition(&tx, offer, &outcome.state, grant)?;
|
|
envelope.payload = Some(Payload::JobResult(jobs::result(
|
|
offer, revision, grant, outcome,
|
|
)));
|
|
jobs::event(&tx, &envelope)?;
|
|
tx.commit()?;
|
|
Ok(())
|
|
}
|
|
/// New process only. Persist honest uncertainty before any helper snapshot
|
|
/// can block; the ordinary first-channel replay publishes it before Sync.
|
|
fn recover_interrupted(&mut self) -> Result<()> {
|
|
let Some(binding) = self.binding.clone() else {
|
|
return Ok(());
|
|
};
|
|
let interrupted = jobs::interrupted(&self.db, &self.ledger, &binding)?;
|
|
for (offer, grant) in interrupted {
|
|
if grant.is_empty() {
|
|
jobs::interrupted_request(&self.db, &offer)?;
|
|
} else {
|
|
let saved =
|
|
jobs::recover_interrupted_outcome(&self.ledger, &binding, &offer, &grant)?;
|
|
if matches!(
|
|
binding.job(&offer.payload_bytes)?.action,
|
|
rmm_domain::jobs::Action::UpdateAgent { .. }
|
|
) && saved.is_none()
|
|
{
|
|
// This original may have intentionally handed off and
|
|
// restarted us. Gates stay closed until a genuine helper
|
|
// capture proves inflight; no Unknown is ever undone.
|
|
continue;
|
|
}
|
|
self.record(
|
|
0,
|
|
&offer,
|
|
&grant,
|
|
saved.unwrap_or_else(|| {
|
|
rmm_privileged_helper::Outcome::unknown("AGENT_PROCESS_INTERRUPTED")
|
|
}),
|
|
)?;
|
|
}
|
|
}
|
|
// No finish/unlock here. Only normal bound reconciliation can release
|
|
// a consumed attempt once independent definitive evidence is available.
|
|
Ok(())
|
|
}
|
|
async fn replay(&mut self, ws: &wire_channel::Handle, generation: u64) -> Result<()> {
|
|
// A large replay backlog must not monopolize heartbeat/cancel handling.
|
|
// On backpressure timeout, discard this channel rather than continuing
|
|
// to use a websocket whose partially sent frame was cancelled.
|
|
let events = jobs::events(&self.db)?;
|
|
tokio::time::timeout(Duration::from_secs(2), async {
|
|
for mut event in events.into_iter().take(8) {
|
|
event.connection_generation = generation;
|
|
ws.send(&event).await?;
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
}
|
|
Ok::<_, anyhow::Error>(())
|
|
})
|
|
.await??;
|
|
Ok(())
|
|
}
|
|
fn permit(
|
|
&mut self,
|
|
generation: u64,
|
|
permit: ExecutionPermit,
|
|
safe_session: bool,
|
|
) -> Result<()> {
|
|
let Some(offer) = jobs::requesting(&self.db)? else {
|
|
anyhow::bail!("UNSOLICITED_PERMIT")
|
|
};
|
|
if !permit.denial_code.is_empty() {
|
|
// A denial is not proof that an earlier reserved grant never existed.
|
|
self.record(
|
|
generation,
|
|
&offer,
|
|
"",
|
|
rmm_privileged_helper::Outcome::unknown("PERMIT_DENIED_RECONCILIATION_REQUIRED"),
|
|
)?;
|
|
return Ok(());
|
|
}
|
|
ensure!(
|
|
safe_session && self.clock.trusted(),
|
|
"CLOCK_OR_SYNC_UNTRUSTED"
|
|
);
|
|
let binding = self
|
|
.binding
|
|
.as_ref()
|
|
.context("job trust not provisioned")?
|
|
.clone();
|
|
let request = jobs::permit_request(&offer, &permit)?;
|
|
let (job, grant) = binding.verify(&request, Utc::now())?;
|
|
ensure!(
|
|
self.ready(generation, job.execution_identity == "root"),
|
|
"FENCED_SYNC_REQUIRED"
|
|
);
|
|
ensure!(
|
|
jobs::supported_with_binding(&job, Some(&binding)),
|
|
"UNSUPPORTED_CAPABILITY"
|
|
);
|
|
rmm_privileged_helper::begin_execution(&mut self.ledger, &grant, &request)?;
|
|
let cancelled: bool = self.db.query_row(
|
|
"SELECT cancel_requested FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
rusqlite::params![offer.job_id, offer.attempt_id],
|
|
|r| r.get(0),
|
|
)?;
|
|
if cancelled {
|
|
let outcome = jobs::no_start("cancelled", "CANCELLED_BEFORE_EFFECT");
|
|
rmm_privileged_helper::persist_outcome(&self.ledger, &grant, &outcome)?;
|
|
self.record(
|
|
generation,
|
|
&offer,
|
|
&grant.grant_id.to_string(),
|
|
outcome.clone(),
|
|
)?;
|
|
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
|
|
return Ok(());
|
|
}
|
|
let mut envelope = self.envelope(
|
|
generation,
|
|
Payload::JobStateChanged(JobStateChanged::default()),
|
|
)?;
|
|
let tx = self.db.transaction()?;
|
|
let revision = jobs::transition(&tx, &offer, "running", &grant.grant_id.to_string())?;
|
|
envelope.payload = Some(Payload::JobStateChanged(JobStateChanged {
|
|
job_id: offer.job_id.clone(),
|
|
attempt_id: offer.attempt_id.clone(),
|
|
revision,
|
|
state: "running".into(),
|
|
grant_id: grant.grant_id.to_string(),
|
|
}));
|
|
jobs::event(&tx, &envelope)?;
|
|
tx.commit()?;
|
|
let cancellation = Arc::new(AtomicBool::new(false));
|
|
self.running_cancels.insert(
|
|
(offer.job_id.clone(), offer.attempt_id.clone()),
|
|
cancellation.clone(),
|
|
);
|
|
let completed = self.completed_tx.clone();
|
|
let clock = self.clock.clone();
|
|
let fence = self.fence.clone();
|
|
tokio::task::spawn_blocking(move || {
|
|
// Thread-pool scheduling and durable fsync may have consumed the
|
|
// remaining start window. Recheck at the actual effect boundary.
|
|
let outcome = if !fence.allows(generation, job.execution_identity == "root")
|
|
|| !clock.trusted()
|
|
|| binding.verify(&request, Utc::now()).is_err()
|
|
{
|
|
jobs::no_start("failed", "START_WINDOW_OR_CLOCK_INVALID_BEFORE_EFFECT")
|
|
} else if job.execution_identity == "root" {
|
|
apt::invoke_with_control(&binding, &job, &request, || {
|
|
ensure!(
|
|
!cancellation.load(Ordering::SeqCst)
|
|
&& fence.running_generation.load(Ordering::SeqCst) == generation
|
|
&& clock.trusted(),
|
|
"SCRIPT_RUNNING_CANCEL_OR_FENCE"
|
|
);
|
|
Ok(())
|
|
})
|
|
.unwrap_or_else(|_| {
|
|
rmm_privileged_helper::Outcome::unknown("HELPER_TRANSPORT_EFFECT_UNKNOWN")
|
|
})
|
|
} else {
|
|
jobs::execute(&job)
|
|
};
|
|
let _ = completed.blocking_send((offer, grant, outcome));
|
|
});
|
|
Ok(())
|
|
}
|
|
fn envelope(&mut self, generation: u64, payload: Payload) -> Result<Envelope> {
|
|
Ok(Envelope {
|
|
protocol_major: 1,
|
|
protocol_minor: 0,
|
|
message_id: Uuid::new_v4().to_string(),
|
|
agent_id: self.id.clone(),
|
|
enrollment_generation: self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
|
connection_generation: generation,
|
|
boot_id: self.boot.clone(),
|
|
sequence: storage::next_sequence(&mut self.db)?,
|
|
observed_at: Utc::now().timestamp_millis(),
|
|
payload: Some(payload),
|
|
})
|
|
}
|
|
fn capabilities(&self) -> Vec<String> {
|
|
let mut capabilities = vec![
|
|
"linux.base/1".into(),
|
|
"heartbeat/1".into(),
|
|
"agent.update/1".into(),
|
|
];
|
|
if self.binding.is_some() {
|
|
capabilities.push("rmm.sync.v1".into());
|
|
}
|
|
if let Some(binding) = &self.binding
|
|
&& unsafe { libc::geteuid() } != 0
|
|
{
|
|
if rmm_agent_linux::actions::Interpreter::PosixSh.available() {
|
|
capabilities.push("run_script".into());
|
|
}
|
|
capabilities.push("refresh_inventory".into());
|
|
if jobs::logs_available() {
|
|
capabilities.push("query_logs".into());
|
|
}
|
|
capabilities.extend(apt::capabilities(binding));
|
|
capabilities.extend(helper_sync::general_capabilities(binding));
|
|
}
|
|
capabilities.sort();
|
|
capabilities.dedup();
|
|
capabilities
|
|
}
|
|
fn queue_inventory(&mut self) {
|
|
if self.inventory_inflight || !storage::passive_collection_ready(&self.db).unwrap_or(false)
|
|
{
|
|
return;
|
|
}
|
|
self.inventory_inflight = true;
|
|
let sender = self.inventory_tx.clone();
|
|
let extended = env::var("RMM_EXTENDED_INVENTORY").unwrap_or_else(|_| "1".into()) == "1";
|
|
let binding = self.binding.clone();
|
|
let catalog = self.catalog.cached(Utc::now());
|
|
tokio::task::spawn_blocking(move || {
|
|
let collected_at = Utc::now();
|
|
let result = rmm_agent_linux::wire::snapshot(extended).and_then(|mut inventory| {
|
|
if extended {
|
|
inventory["observations"]
|
|
.as_array_mut()
|
|
.context("INVENTORY_OBSERVATIONS")?
|
|
.push(apt::planning_observation(binding.as_ref()));
|
|
inventory["collection_completed_at"] = serde_json::json!(Utc::now());
|
|
}
|
|
if let Some(report) = catalog {
|
|
if inventory.get("observations").is_none() {
|
|
inventory["observations"] = serde_json::json!([]);
|
|
inventory["collection_started_at"] = serde_json::json!(collected_at);
|
|
}
|
|
inventory["observations"]
|
|
.as_array_mut()
|
|
.context("INVENTORY_OBSERVATIONS")?
|
|
.push(agent_update_catalog::observation(report, Utc::now()));
|
|
inventory["collection_completed_at"] = serde_json::json!(Utc::now());
|
|
}
|
|
let bytes = serde_json::to_vec(&inventory)?;
|
|
ensure!(bytes.len() <= 512 * 1024, "INVENTORY_WIRE_LIMIT");
|
|
Ok((bytes, Utc::now().timestamp_millis()))
|
|
});
|
|
let _ = sender.blocking_send(result);
|
|
});
|
|
}
|
|
fn queue_update_observation(&mut self, generation: u64) -> Result<()> {
|
|
if !storage::passive_collection_ready(&self.db)? {
|
|
return Ok(());
|
|
}
|
|
// Until the separately qualified protected updater/helper observer is
|
|
// provisioned, report its real absence; never infer apply readiness from
|
|
// this schema support or the agent's own code/build version.
|
|
let observation = rmm_domain::agent_update::AgentUpdateObservation::not_configured(
|
|
rmm_domain::agent_update::Platform::LinuxX64,
|
|
Utc::now(),
|
|
);
|
|
let address = self.address;
|
|
let ca = &self.ca;
|
|
let certificates = &self.certificates;
|
|
self.catalog
|
|
.poll(self.binding.as_ref(), self.clock.clone(), address, || {
|
|
let identity = certificates.current(Utc::now().timestamp())?;
|
|
https_channel::mtls_client(
|
|
address,
|
|
ca,
|
|
&identity.certificate_pem,
|
|
&identity.private_key_pem,
|
|
)
|
|
});
|
|
let data = serde_json::to_vec(&serde_json::json!({"agent_update":observation}))?;
|
|
let envelope = self.envelope(
|
|
generation,
|
|
Payload::InventorySnapshot(rmm_protocol::InventorySnapshot {
|
|
snapshot_id: Uuid::new_v4().to_string(),
|
|
scope: "agent.update".into(),
|
|
schema_version: 1,
|
|
chunk_index: 0,
|
|
chunk_count: 1,
|
|
sha256: Sha256::digest(&data).to_vec(),
|
|
data,
|
|
completeness: "partial".into(),
|
|
}),
|
|
)?;
|
|
jobs::event(&self.db, &envelope)
|
|
}
|
|
fn inventory_envelope(
|
|
&mut self,
|
|
generation: u64,
|
|
data: Vec<u8>,
|
|
observed_at: i64,
|
|
) -> Result<Envelope> {
|
|
let mut envelope = self.envelope(
|
|
generation,
|
|
Payload::InventorySnapshot(InventorySnapshot {
|
|
snapshot_id: Uuid::new_v4().to_string(),
|
|
scope: "linux.base".into(),
|
|
schema_version: 1,
|
|
chunk_index: 0,
|
|
chunk_count: 1,
|
|
sha256: Sha256::digest(&data).to_vec(),
|
|
data,
|
|
completeness: "partial".into(),
|
|
}),
|
|
)?;
|
|
envelope.observed_at = observed_at;
|
|
Ok(envelope)
|
|
}
|
|
async fn session(&mut self) -> Result<()> {
|
|
let result = self.session_inner().await;
|
|
self.close_gate();
|
|
result
|
|
}
|
|
async fn session_inner(&mut self) -> Result<()> {
|
|
self.close_gate();
|
|
self.boot = rmm_agent_linux::boot_id()?;
|
|
if self.helpers.is_some() {
|
|
self.helpers = self
|
|
.binding
|
|
.as_ref()
|
|
.map(helper_sync::Helpers::configured)
|
|
.transpose()?;
|
|
}
|
|
let advertised = self.capabilities();
|
|
let hello = self.envelope(
|
|
0,
|
|
Payload::ClientHello(ClientHello {
|
|
agent_version: env!("CARGO_PKG_VERSION").into(),
|
|
capabilities: advertised.clone(),
|
|
}),
|
|
)?;
|
|
let identity = self.certificates.current(Utc::now().timestamp())?;
|
|
let mut ws =
|
|
wire_channel::connect(self.mode, self.address, &self.ca, &identity, hello).await?;
|
|
let reply = tokio::time::timeout(Duration::from_secs(20), ws.receive()).await??;
|
|
ensure!(
|
|
reply.protocol_major == 1
|
|
&& reply.protocol_minor == 0
|
|
&& reply.agent_id == self.id
|
|
&& reply.enrollment_generation
|
|
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
|
"invalid server hello"
|
|
);
|
|
let Some(Payload::ServerHello(server)) = reply.payload else {
|
|
anyhow::bail!("server hello required")
|
|
};
|
|
let generation = server.connection_generation;
|
|
ensure!(
|
|
generation > 0 && server.max_message_bytes == rmm_domain::MAX_MESSAGE as u32,
|
|
"unsupported limits"
|
|
);
|
|
if let Some(mut pending) = storage::pending(&self.db)? {
|
|
// Generation is transport fencing, not part of durable message identity.
|
|
pending.connection_generation = generation;
|
|
jobs::event(&self.db, &pending)?;
|
|
storage::clear_pending(&self.db)?;
|
|
}
|
|
self.queue_inventory();
|
|
self.queue_update_observation(generation)?;
|
|
self.replay(&ws, generation).await?;
|
|
let mut needs_sync = false;
|
|
if self.sync.is_some()
|
|
&& let Err(error) = self.start_sync(&ws, generation).await
|
|
{
|
|
report_closed_gate(&error);
|
|
self.close_gate();
|
|
needs_sync = true;
|
|
}
|
|
eprintln!("authenticated channel established; inventory queued for durable acknowledgment");
|
|
let mut inventory_at = Instant::now();
|
|
let mut update_at = Instant::now();
|
|
let mut heartbeat_at = Instant::now();
|
|
let mut heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
|
let mut tick = tokio::time::interval(Duration::from_secs(5));
|
|
let mut sync_at = Instant::now();
|
|
let mut rotation_at = Instant::now();
|
|
let mut capability_at = Instant::now();
|
|
let mut first_heartbeat = true;
|
|
let mut health_task: Option<PendingUpdateHealth> = None;
|
|
let mut health_inventory_prefix: Option<String> = None;
|
|
loop {
|
|
tokio::select! {
|
|
Some(result) = self.inventory_rx.recv() => {
|
|
self.inventory_inflight = false;
|
|
if let Ok((data,observed_at)) = result {
|
|
let inventory = self.inventory_envelope(generation,data,observed_at)?;
|
|
jobs::event(&self.db,&inventory)?;
|
|
if let Some(binding)=&self.binding
|
|
&& let Err(error)=rmm_agent_core::update_health_inventory::retain_current(&self.db,&inventory,binding) {
|
|
report_closed_gate(&error);self.close_gate();needs_sync=true;
|
|
}
|
|
} else { eprintln!("inventory collection failed; previous observations remain historical"); }
|
|
}
|
|
Some(result)=async {
|
|
match health_task.as_mut() {Some(task)=>Some((&mut task.0).await),None=>std::future::pending().await}
|
|
} => {
|
|
health_task.take();
|
|
if let Err(error)=result.map_err(anyhow::Error::from).and_then(std::convert::identity) {report_closed_gate(&error);}
|
|
self.close_gate();needs_sync=true;
|
|
},
|
|
message = ws.receive() => {
|
|
let message = message?;
|
|
ensure!(message.protocol_major == 1 && message.protocol_minor==0 && message.enrollment_generation==self.binding.as_ref().map_or(1,|b|b.enrollment_generation) && message.agent_id == self.id && message.connection_generation == generation, "INVALID_SERVER_ENVELOPE");
|
|
match message.payload {
|
|
Some(Payload::Ack(ack)) => {
|
|
use rusqlite::OptionalExtension;
|
|
let pending:Option<Vec<u8>>=self.db.query_row("SELECT payload FROM job_events WHERE id=?1",[&ack.message_id],|r|r.get(0)).optional()?;
|
|
if let Some(bytes)=pending && matches!(Envelope::decode(bytes.as_slice())?.payload,Some(Payload::InventorySnapshot(_)|Payload::JobResult(_))){needs_sync=true;}
|
|
jobs::ack(&mut self.db, &ack.message_id)?;
|
|
},
|
|
Some(Payload::SyncResponse(response))=>{if let Err(error)=self.apply_sync(generation,response).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}sync_at=Instant::now();},
|
|
Some(Payload::JobOffer(offer)) => {
|
|
let binding = self.binding.as_ref().context("job trust not provisioned")?.clone();
|
|
let accepted = self.envelope(generation, Payload::JobAccepted(JobAccepted {
|
|
job_id: offer.job_id.clone(), attempt_id: offer.attempt_id.clone(),
|
|
job_payload_hash: offer.job_payload_hash.clone(), journal_proof: rmm_domain::jobs::hash(&offer.encode_to_vec()),
|
|
}))?;
|
|
// Clone before borrowing the transaction; acceptance and the
|
|
// durable ACK event commit together, before any wire ACK.
|
|
let tx = self.db.transaction()?;
|
|
let fresh=jobs::accept(&tx, &binding, &offer)?;
|
|
jobs::event(&tx, &accepted)?;
|
|
tx.commit()?;
|
|
needs_sync|=fresh;self.replay(&ws, generation).await?;
|
|
}
|
|
Some(Payload::ExecutionPermit(permit)) => self.permit(generation, permit, self.ready(generation,false))?,
|
|
Some(Payload::CancelRequest(cancel)) => {
|
|
let row: Option<(Vec<u8>, String, String)> = {
|
|
use rusqlite::OptionalExtension;
|
|
self.db.query_row("SELECT offer,state,grant_id FROM job_attempts WHERE job=?1 AND attempt=?2",
|
|
rusqlite::params![cancel.job_id,cancel.attempt_id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?
|
|
};
|
|
if let Some((bytes,state,grant)) = row {
|
|
if let Some(flag) = self.running_cancels.get(&(cancel.job_id.clone(),cancel.attempt_id.clone())) { flag.store(true,Ordering::SeqCst); }
|
|
self.db.execute("UPDATE job_attempts SET cancel_requested=1 WHERE job=?1 AND attempt=?2", rusqlite::params![cancel.job_id,cancel.attempt_id])?;
|
|
if state == "accepted" { self.record(generation, &JobOffer::decode(bytes.as_slice())?, &grant, jobs::no_start("cancelled", "CANCELLED_BEFORE_START"))?; }
|
|
// Running script cancellation is not confirmed by an
|
|
// inbound request. Its bounded executor reports reality.
|
|
}
|
|
let ack = self.envelope(generation, Payload::Ack(Ack { message_id: message.message_id }))?;
|
|
ws.send(&ack).await?;
|
|
}
|
|
_ => anyhow::bail!("UNSUPPORTED_SERVER_MESSAGE"),
|
|
}
|
|
}
|
|
Some((offer, grant, outcome)) = self.completed_rx.recv() => {
|
|
self.running_cancels.remove(&(offer.job_id.clone(),offer.attempt_id.clone()));
|
|
// First preserve the original completion without unlocking.
|
|
// A startup can replay it if the journal transaction is lost.
|
|
let state = outcome.state.clone();
|
|
let refresh_capabilities=state=="succeeded" && serde_json::from_slice::<rmm_domain::jobs::JobPayload>(&offer.payload_bytes)?.action.name()=="apt_native_acceptance";
|
|
rmm_privileged_helper::persist_outcome(&self.ledger,&grant,&outcome)?;
|
|
let serialized = serde_json::to_vec(&outcome)?;
|
|
self.record(generation, &offer, &grant.grant_id.to_string(), outcome)?;
|
|
let outcome: rmm_privileged_helper::Outcome = serde_json::from_slice(&serialized)?;
|
|
rmm_privileged_helper::finish(&self.ledger, &grant, &outcome)?;
|
|
if state == "unknown" { eprintln!("job effect unknown: device execution lock retained"); }
|
|
needs_sync=true;self.replay(&ws, generation).await?;
|
|
if refresh_capabilities {return Ok(());}
|
|
}
|
|
_ = tick.tick() => {
|
|
self.replay(&ws, generation).await?;
|
|
let busy:bool=self.db.query_row("SELECT EXISTS(SELECT 1 FROM job_attempts WHERE state IN ('requesting','running','verifying'))",[],|r|r.get(0))?;
|
|
let update=self.update_proof()?;
|
|
let pending_update=self.binding.as_ref().map(|binding|rmm_agent_core::jobs::needs_update_control(&self.db,&self.ledger,binding)).transpose()?.unwrap_or(false);
|
|
let own_update=update.is_some() || pending_update;
|
|
if let Some(proof)=&update {
|
|
let decoded=proof.decode().map_err(anyhow::Error::msg)?;
|
|
let event:Option<rmm_updater_handoff::Event>=decoded.events.last().map(|raw|serde_json::from_slice(raw)).transpose()?;
|
|
if event.is_some_and(|event|matches!(event.evidence,rmm_updater_handoff::Evidence::ServicesStarted{..})) {
|
|
if health_inventory_prefix.as_deref()!=Some(proof.head_sha256.as_str()) {
|
|
// Fixed update health includes one genuinely new
|
|
// normal base sample, never a fresh timestamp on old data.
|
|
self.queue_inventory();health_inventory_prefix=Some(proof.head_sha256.clone());
|
|
}
|
|
if health_task.is_none() && self.prepared.is_none() {
|
|
match self.start_update_health(proof) {Ok(task)=>health_task=task,Err(error)=>{report_closed_gate(&error);self.close_gate();needs_sync=true;}}
|
|
}
|
|
}
|
|
}
|
|
if !busy && capability_at.elapsed()>=Duration::from_secs(60) {capability_at=Instant::now();if self.capabilities()!=advertised{return Ok(());}}
|
|
if self.prepared.is_some() && (!busy||own_update) && health_task.is_none() && sync_at.elapsed()>=Duration::from_secs(20){self.close_gate();needs_sync=true;}
|
|
if self.prepared.is_none() && self.sync.is_some() && (!busy||own_update) && health_task.is_none() && (needs_sync||sync_at.elapsed()>=Duration::from_secs(40)||!self.ready(generation,false)) {
|
|
if let Err(error)=self.start_sync(&ws,generation).await{report_closed_gate(&error);self.close_gate();needs_sync=true;}else{needs_sync=false;}sync_at=Instant::now();
|
|
}
|
|
if !needs_sync && self.prepared.is_none() && self.ready(generation,false) && jobs::events(&self.db)?.is_empty() && self.binding.is_some()
|
|
&& self.clock.trusted() && let Some(offer) = jobs::next(&self.db)? {
|
|
let job = self.binding.as_ref().context("binding")?.job(&offer.payload_bytes)?;
|
|
if Utc::now() >= job.start_before {
|
|
self.record(generation, &offer, "", jobs::no_start("expired", "START_DEADLINE_EXPIRED"))?;
|
|
} else if !self.ready(generation,job.execution_identity=="root") {
|
|
needs_sync=true;
|
|
} else if Utc::now() >= job.not_before {
|
|
// Reserve before send. A lost response must never cause
|
|
// a second permit request for this attempt.
|
|
jobs::reserve_request(&self.db, &offer)?;
|
|
let request = self.envelope(generation, Payload::ExecutionPermitRequest(ExecutionPermitRequest {
|
|
job_id: offer.job_id, attempt_id: offer.attempt_id,
|
|
job_payload_hash: offer.job_payload_hash, assignment_version: job.assignment_version,
|
|
}))?;
|
|
ws.send(&request).await?;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if first_heartbeat || heartbeat_at.elapsed() >= heartbeat_after {
|
|
first_heartbeat = false;
|
|
let heartbeat = self.envelope(
|
|
generation,
|
|
Payload::Heartbeat(Heartbeat {
|
|
instance_id: self.instance.clone(),
|
|
uptime_seconds: self.started.elapsed().as_secs(),
|
|
}),
|
|
)?;
|
|
ws.send(&heartbeat).await?;
|
|
if ws.uses_https() {
|
|
let poll = self.envelope(
|
|
generation,
|
|
Payload::Heartbeat(Heartbeat {
|
|
instance_id: self.instance.clone(),
|
|
uptime_seconds: self.started.elapsed().as_secs(),
|
|
}),
|
|
)?;
|
|
ws.poll(poll).await?;
|
|
}
|
|
heartbeat_at = Instant::now();
|
|
heartbeat_after = Duration::from_millis(rand::random_range(27_000..33_000));
|
|
}
|
|
// Planning pins must be refreshed before the 90-second stored-data
|
|
// planning limit. One inflight blocking collector bounds pressure;
|
|
// heartbeats never perform this metadata work.
|
|
if update_at.elapsed() >= Duration::from_secs(60) {
|
|
self.queue_update_observation(generation)?;
|
|
update_at = Instant::now();
|
|
}
|
|
if inventory_at.elapsed() >= Duration::from_secs(60) {
|
|
self.queue_inventory();
|
|
inventory_at = Instant::now();
|
|
}
|
|
if rotation_at.elapsed() >= Duration::from_secs(30) {
|
|
rotation_at = Instant::now();
|
|
if self
|
|
.certificates
|
|
.due(self.rotation_policy, Utc::now().timestamp())?
|
|
&& self.update_proof()?.is_none()
|
|
{
|
|
self.close_gate();
|
|
drop(ws);
|
|
let rotated = self
|
|
.certificates
|
|
.renew(self.address, Utc::now().timestamp())
|
|
.await?;
|
|
ensure!(
|
|
rotated.needs_reconnect
|
|
&& rotated.enrollment_generation
|
|
== self.binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
|
"ROTATION_IDENTITY_CHANGED"
|
|
);
|
|
return Ok(());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn enrollment(dir: &Path, address: std::net::SocketAddr, ca: &[u8]) -> Result<String> {
|
|
let identity = dir.join("identity.json");
|
|
if identity.exists() {
|
|
let response: EnrollmentResponse = serde_json::from_slice(&fs::read(identity)?)?;
|
|
Uuid::parse_str(&response.agent_id)?;
|
|
return Ok(response.agent_id);
|
|
}
|
|
let key_path = dir.join("device.key");
|
|
let key = if key_path.exists() {
|
|
PKey::private_key_from_pem(&fs::read(&key_path)?)?
|
|
} else {
|
|
let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1)?;
|
|
let key = PKey::from_ec_key(EcKey::generate(&group)?)?;
|
|
storage::create_secret(&key_path, &key.private_key_to_pem_pkcs8()?)?;
|
|
key
|
|
};
|
|
let mut csr = X509Req::builder()?;
|
|
let mut name = X509NameBuilder::new()?;
|
|
name.append_entry_by_text("CN", "new-linux-device")?;
|
|
csr.set_subject_name(&name.build())?;
|
|
csr.set_pubkey(&key)?;
|
|
csr.sign(&key, MessageDigest::sha256())?;
|
|
let token_path = env::var("RMM_ENROLLMENT_TOKEN_FILE")
|
|
.context("enrollment token file required for first registration")?;
|
|
let token = fs::read_to_string(token_path)?.trim().to_owned();
|
|
ensure!(token.len() == 64, "invalid enrollment token format");
|
|
let client = reqwest::Client::builder()
|
|
.https_only(true)
|
|
.tls_built_in_root_certs(false)
|
|
.add_root_certificate(reqwest::Certificate::from_pem(ca)?)
|
|
.resolve(&https_channel::server_name()?, address)
|
|
.no_proxy()
|
|
.redirect(reqwest::redirect::Policy::none())
|
|
.timeout(Duration::from_secs(15))
|
|
.build()?;
|
|
let result = client
|
|
.post(format!(
|
|
"https://{}:{}/agent/v1/enroll",
|
|
https_channel::server_name()?,
|
|
address.port()
|
|
))
|
|
.json(&EnrollmentRequest {
|
|
token,
|
|
csr_pem: String::from_utf8(csr.build().to_pem()?)?,
|
|
})
|
|
.send()
|
|
.await?;
|
|
ensure!(
|
|
result.status().is_success(),
|
|
"enrollment rejected (response details suppressed)"
|
|
);
|
|
let response: EnrollmentResponse = result.json().await?;
|
|
let certificate = openssl::x509::X509::from_pem(response.certificate_pem.as_bytes())?;
|
|
ensure!(
|
|
certificate.public_key()?.public_eq(&key),
|
|
"enrollment certificate key mismatch"
|
|
);
|
|
Uuid::parse_str(&response.agent_id)?;
|
|
storage::create_secret(&dir.join("device.pem"), response.certificate_pem.as_bytes())?;
|
|
storage::create_secret(&identity, &serde_json::to_vec(&response)?)?;
|
|
Ok(response.agent_id)
|
|
}
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let args: Vec<_> = env::args_os().skip(1).collect();
|
|
if args == [std::ffi::OsString::from("--help")] {
|
|
println!(
|
|
"rmm-agent [--help]\nEnvironment: RMM_AGENT_STATE, RMM_CA_FILE, RMM_CONNECT_ADDR (TCP only; TLS name rmm-dev.internal), RMM_TRANSPORT=auto|wss|https, RMM_ENROLLMENT_TOKEN_FILE (first enrollment), RMM_JOB_BINDING_FILE, RMM_JOB_SIGNING_KEY_BOOTSTRAP=1, RMM_HELPER_SOCKET, RMM_HELPER_POLICY_FILE, RMM_APT_HELPER_SOCKET, RMM_APT_HELPER_PROOF_FILE, RMM_LOG_UNITS_FILE, RMM_EXTENDED_INVENTORY=0|1 (default 1).\nDevelopment-only rotation test: RMM_AGENT_PROFILE=development and RMM_ROTATION_TEST_AFTER_SECONDS.\nNo argument or environment variable approves jobs or disables TLS/grant verification."
|
|
);
|
|
return Ok(());
|
|
}
|
|
ensure!(args.is_empty(), "UNKNOWN_AGENT_ARGUMENT");
|
|
let dir =
|
|
PathBuf::from(env::var("RMM_AGENT_STATE").unwrap_or_else(|_| "devstate/agent".into()));
|
|
storage::prepare_dir(&dir)?;
|
|
let _lock = storage::lock(&dir)?;
|
|
let address = env::var("RMM_CONNECT_ADDR")
|
|
.unwrap_or_else(|_| "127.0.0.1:18443".into())
|
|
.parse()?;
|
|
let ca = fs::read(env::var("RMM_CA_FILE").unwrap_or_else(|_| "devstate/pki/ca.pem".into()))?;
|
|
let id=enrollment(&dir,address,&ca).await.map_err(|_| anyhow::anyhow!("enrollment/identity initialization failed; check trusted CA, token validity and local state"))?;
|
|
let mut binding = env::var("RMM_JOB_BINDING_FILE")
|
|
.ok()
|
|
.map(|path| {
|
|
let binding =
|
|
rmm_privileged_helper::read_binding(Path::new(&path), unsafe { libc::geteuid() })?;
|
|
let identity: EnrollmentResponse =
|
|
serde_json::from_slice(&fs::read(dir.join("identity.json"))?)?;
|
|
ensure!(
|
|
binding.agent_id.to_string() == id
|
|
&& binding.asset_id.to_string() == identity.asset_id,
|
|
"BOOTSTRAP_IDENTITY_MISMATCH"
|
|
);
|
|
Ok::<_, anyhow::Error>(binding)
|
|
})
|
|
.transpose()?;
|
|
let certificate_dir = dir.join("certificates");
|
|
let certificates = certificate_rotation::RotationStore::open(
|
|
&certificate_dir,
|
|
&ca,
|
|
Uuid::parse_str(&id)?,
|
|
binding.as_ref().map_or(1, |b| b.enrollment_generation),
|
|
)?;
|
|
if !certificate_dir.join("current.json").try_exists()? {
|
|
certificates.initialize(
|
|
&fs::read(dir.join("device.pem"))?,
|
|
&fs::read(dir.join("device.key"))?,
|
|
Utc::now().timestamp(),
|
|
)?;
|
|
}
|
|
if env::var("RMM_JOB_SIGNING_KEY_BOOTSTRAP").as_deref() == Ok("1") {
|
|
bootstrap::provision_current(
|
|
&dir,
|
|
address,
|
|
&ca,
|
|
binding
|
|
.as_mut()
|
|
.context("bootstrap requires provisioned device binding")?,
|
|
&certificates.current(Utc::now().timestamp())?,
|
|
)
|
|
.await?;
|
|
}
|
|
let (completed_tx, completed_rx) = tokio::sync::mpsc::channel(1);
|
|
let (inventory_tx, inventory_rx) = tokio::sync::mpsc::channel(1);
|
|
let sync = binding
|
|
.as_ref()
|
|
.map(|b| reconcile::ClientSync::new(b.clone()));
|
|
let helpers = binding
|
|
.as_ref()
|
|
.map(helper_sync::Helpers::configured)
|
|
.transpose()?;
|
|
let mut rotation_policy = certificate_rotation::RotationPolicy::default();
|
|
if let Ok(after) = env::var("RMM_ROTATION_TEST_AFTER_SECONDS") {
|
|
ensure!(
|
|
env::var("RMM_AGENT_PROFILE").as_deref() == Ok("development"),
|
|
"ROTATION_TEST_REQUIRES_DEVELOPMENT_PROFILE"
|
|
);
|
|
rotation_policy.begin_after_seconds = after.parse()?;
|
|
ensure!(
|
|
rotation_policy.begin_after_seconds > 0,
|
|
"ROTATION_TEST_INTERVAL"
|
|
);
|
|
}
|
|
let mode =
|
|
wire_channel::Mode::parse(&env::var("RMM_TRANSPORT").unwrap_or_else(|_| "auto".into()))?;
|
|
let mut agent = Agent {
|
|
address,
|
|
ca,
|
|
id,
|
|
boot: rmm_agent_linux::boot_id()?,
|
|
instance: Uuid::new_v4().to_string(),
|
|
db: storage::open(&dir)?,
|
|
started: Instant::now(),
|
|
clock: rmm_privileged_helper::TrustedClock::new(),
|
|
binding,
|
|
ledger: rmm_privileged_helper::open_ledger(&dir, unsafe { libc::geteuid() })?,
|
|
completed_tx,
|
|
completed_rx,
|
|
inventory_tx,
|
|
inventory_rx,
|
|
inventory_inflight: false,
|
|
running_cancels: std::collections::HashMap::new(),
|
|
sync,
|
|
sync_clock: reconcile::NativeClock::default(),
|
|
helpers,
|
|
prepared: None,
|
|
fence: Arc::new(EffectFence::default()),
|
|
mode,
|
|
certificates,
|
|
rotation_policy,
|
|
catalog: agent_update_catalog::Checker::default(),
|
|
};
|
|
agent.recover_interrupted()?;
|
|
let mut backoff = 1_u64;
|
|
loop {
|
|
let started = Instant::now();
|
|
if let Err(error) = agent.session().await {
|
|
report_closed_gate(&error);
|
|
}
|
|
agent.close_gate();
|
|
eprintln!("channel unavailable; retrying with jitter (no jobs can execute)");
|
|
if started.elapsed() > Duration::from_secs(60) {
|
|
backoff = 1;
|
|
}
|
|
tokio::time::sleep(https_channel::retry_delay(backoff as u32)).await;
|
|
backoff = (backoff + 1).min(6);
|
|
}
|
|
}
|