Files
rust-discovery/crates/server/src/windows_echo_probe.rs
T
2026-11-16 10:25:25 +01:00

520 lines
20 KiB
Rust

//! Bind the development checkpoint intervention to a single immutable LocalService echo.
//! The scope outlives its signed transport receipt and is also enforced by PostgreSQL.
use crate::{
auth::Principal,
jobs::Error,
windows_checkpoint_review::{access, verify_probe_case},
};
use axum::{
Json, Router,
extract::{Path, State},
http::{HeaderMap, StatusCode},
routing::{get, post},
};
use chrono::{DateTime, Duration, Utc};
use openssl::{
pkey::{Id, PKey},
sign::Verifier,
};
use rmm_domain::{
checkpoint_repair::{SignedReceipt, inventory_sha256},
jobs::{PlanPayload, encode, hash},
windows_echo_probe::{AGENT, ASSET, DOMAIN, ProbeReceipt, ProbeScope, TENANT, TEST_PRINCIPAL},
windows_sync_history::case_witness,
};
use serde::Deserialize;
use serde_json::{Value, json};
use sqlx::{Postgres, Row, Transaction};
use uuid::Uuid;
type Result<T> = std::result::Result<T, Error>;
fn mismatch() -> Error {
Error(StatusCode::CONFLICT, "WINDOWS_ECHO_SCOPE_MISMATCH")
}
fn forbidden() -> Error {
Error(StatusCode::FORBIDDEN, "DEV_TEST_WEBUI_REQUIRED")
}
fn missing() -> Error {
Error(StatusCode::NOT_FOUND, "NOT_FOUND")
}
pub fn routes(r: Router<crate::State>) -> Router<crate::State> {
r.route(
"/api/v1/tenants/{tenant}/windows-echo-probe/{operation}",
get(read),
)
.route(
"/api/v1/tenants/{tenant}/windows-echo-probe/{operation}/confirm",
post(confirm),
)
.route(
"/api/v1/tenants/{tenant}/windows-echo-probe/{operation}/receipt",
get(download),
)
}
fn browser(p: &Principal) -> Result<&str> {
if p.name != TEST_PRINCIPAL
|| p.tenant != TENANT
|| !p.testsimulation
|| !p.operator
|| p.identity.is_some()
|| p.scopes.is_some()
|| p.csrf.is_none()
{
return Err(forbidden());
}
p.session_hash.as_deref().ok_or_else(forbidden)
}
async fn reviewer(tx: &mut Transaction<'_, Postgres>, p: &Principal) -> Result<()> {
browser(p)?;
let allowed: bool = sqlx::query_scalar("SELECT lock_windows_echo_probe_reviewer()")
.fetch_one(&mut **tx)
.await?;
if !allowed {
return Err(forbidden());
}
Ok(())
}
struct Review {
scope: ProbeScope,
raw_job: Vec<u8>,
request_id: Uuid,
witness: Value,
witness_hash: String,
scope_hash: String,
existing: Option<(Uuid, DateTime<Utc>, SignedReceipt)>,
}
impl Review {
fn view(&self) -> Value {
json!({"scope":self.scope,"scope_sha256":self.scope_hash,"witness_sha256":self.witness_hash,"witness":self.witness,
"receipt_id":self.existing.as_ref().map(|r|r.0),"valid_until":self.existing.as_ref().map(|r|r.1)})
}
}
async fn review(
tx: &mut Transaction<'_, Postgres>,
s: &crate::State,
op: Uuid,
expected: &Value,
) -> Result<Review> {
// Same control lock ordering as jobs and checkpoint recovery.
crate::agent_service::control_lock(tx, TENANT, AGENT).await?;
let r = sqlx::query(
"SELECT o.payload,o.plan_hash,a.approval_state,w.request_id,w.witness,w.witness_sha256
FROM approval_requests a JOIN operation_plans o USING(tenant_id,plan_id)
JOIN windows_checkpoint_reviews w ON w.tenant_id=a.tenant_id AND w.agent_id=$3
WHERE a.tenant_id=$1 AND a.operation_id=$2 FOR UPDATE OF a",
)
.bind(TENANT)
.bind(op)
.bind(AGENT)
.fetch_optional(&mut **tx)
.await?
.ok_or_else(missing)?;
if r.get::<String, _>("approval_state") != "pending_approval" {
return Err(mismatch());
}
let plan_bytes: Vec<u8> = r.get("payload");
let plan: PlanPayload = serde_json::from_slice(&plan_bytes)?;
let scope = ProbeScope::from_plan(&plan, op).ok_or_else(mismatch)?;
if scope.plan_hash != r.get::<String, _>("plan_hash")
|| hash(&plan_bytes) != scope.plan_hash
|| scope.start_before <= Utc::now()
{
return Err(mismatch());
}
let witness: Value = r.get("witness");
let witness_hash = inventory_sha256(&witness)?;
if &witness != expected || witness_hash != r.get::<String, _>("witness_sha256") {
return Err(mismatch());
}
let scope_hash = scope.digest().ok_or_else(mismatch)?;
let stored=sqlx::query("SELECT operation_id,receipt_id,valid_until,receipt,signature FROM windows_echo_probe_scopes WHERE tenant_id=$1")
.bind(TENANT).fetch_optional(&mut **tx).await?;
let mut existing = None;
if let Some(old) = stored {
if old.get::<Uuid, _>("operation_id") != op {
return Err(mismatch());
}
let signed = SignedReceipt {
payload: old.get("receipt"),
signature: old.get("signature"),
};
let receipt: ProbeReceipt = serde_json::from_slice(&signed.payload)?;
let key = PKey::public_key_from_raw_bytes(&s.job_signer.public_key()?, Id::ED25519)
.map_err(|_| mismatch())?;
if !Verifier::new_without_digest(&key)
.map_err(|_| mismatch())?
.verify_oneshot(&signed.signature, &signed.payload)
.map_err(|_| mismatch())?
|| !receipt.matches_reviewed_witness(expected, Utc::now())
|| receipt.scope != scope
|| receipt.receipt_id != old.get::<Uuid, _>("receipt_id")
|| receipt.request_id != r.get::<Uuid, _>("request_id")
|| receipt.valid_until.timestamp_micros()
!= old
.get::<DateTime<Utc>, _>("valid_until")
.timestamp_micros()
{
return Err(mismatch());
}
existing = Some((receipt.receipt_id, receipt.valid_until, signed));
}
verify_probe_case(tx, s, &witness, existing.is_some()).await?;
Ok(Review {
raw_job: encode(&plan.jobs[0].payload)?,
scope,
scope_hash,
request_id: r.get("request_id"),
witness,
witness_hash,
existing,
})
}
async fn read(
State(s): State<crate::State>,
Path((tenant, op)): Path<(Uuid, Uuid)>,
h: HeaderMap,
) -> Result<Json<Value>> {
let p = access(&s, &h, tenant, false).await?;
let mut tx = rmm_persistence::tenant_tx(&s.pool, tenant).await?;
crate::permissions::require_principal(&mut tx, &p, "operations:read").await?;
let info = review(&mut tx, &s, op, &case_witness()).await?.view();
tx.commit().await?;
Ok(Json(info))
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Confirmation {
reviewed: bool,
scope_sha256: String,
witness_sha256: String,
}
async fn confirm(
State(s): State<crate::State>,
Path((tenant, op)): Path<(Uuid, Uuid)>,
h: HeaderMap,
Json(input): Json<Confirmation>,
) -> Result<Json<Value>> {
let p = access(&s, &h, tenant, true).await?;
confirm_review(&s, &p, op, input, &case_witness()).await
}
async fn confirm_review(
s: &crate::State,
p: &Principal,
op: Uuid,
input: Confirmation,
expected: &Value,
) -> Result<Json<Value>> {
let session = browser(p)?;
if !input.reviewed {
return Err(Error(StatusCode::BAD_REQUEST, "REVIEW_REQUIRED"));
}
let mut tx = rmm_persistence::tenant_tx(&s.pool, p.tenant).await?;
reviewer(&mut tx, p).await?;
let mut r = review(&mut tx, s, op, expected).await?;
if input.scope_sha256 != r.scope_hash || input.witness_sha256 != r.witness_hash {
return Err(mismatch());
}
if r.existing.is_none() {
let now = Utc::now();
let receipt = ProbeReceipt {
domain: DOMAIN.into(),
schema_version: 1,
request_id: r.request_id,
receipt_id: Uuid::new_v4(),
witness: r.witness.clone(),
witness_sha256: r.witness_hash.clone(),
scope: r.scope.clone(),
scope_sha256: r.scope_hash.clone(),
job_payload_bytes: r.raw_job.clone(),
key_id: s.job_signer.key_id.clone(),
issued_at: now,
valid_until: std::cmp::min(now + Duration::minutes(10), r.scope.start_before),
confirmed_by: p.name.clone(),
approval_origin: p.approval_origin().into(),
testsimulation: true,
};
if !receipt.matches_reviewed_witness(expected, now) {
return Err(mismatch());
}
let raw = serde_json::to_vec(&receipt)?;
let signed = SignedReceipt {
signature: s.job_signer.sign(&raw)?,
payload: raw,
};
sqlx::query("INSERT INTO windows_echo_probe_scopes(tenant_id,asset_id,agent_id,operation_id,plan_id,plan_hash,job_id,attempt_id,job_bytes,job_hash,request_id,receipt_id,receipt,signature,confirmed_by,session_hash,issued_at,valid_until)
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)")
.bind(TENANT).bind(ASSET).bind(AGENT).bind(op).bind(r.scope.plan_id).bind(&r.scope.plan_hash).bind(r.scope.job_id).bind(r.scope.attempt_id)
.bind(&r.raw_job).bind(&r.scope.job_payload_hash).bind(r.request_id).bind(receipt.receipt_id).bind(&signed.payload).bind(&signed.signature)
.bind(&p.name).bind(session).bind(now).bind(receipt.valid_until).execute(&mut *tx).await?;
// Only after installing the immutable scope in the same transaction.
if sqlx::query("UPDATE agent_sync_state SET checkpoint=6756,decision_id=$4 WHERE tenant_id=$1 AND agent_id=$2 AND checkpoint=6755 AND decision_id=$3 AND helper_checkpoint=1")
.bind(TENANT).bind(AGENT).bind(crate::windows_checkpoint_review::FIRST).bind(crate::windows_checkpoint_review::DENIED)
.execute(&mut *tx).await?.rows_affected()!=1{return Err(mismatch());}
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,'windows_echo_probe_scoped',$4)")
.bind(TENANT).bind(receipt.receipt_id).bind(&p.name).bind(json!({"scope":r.scope,"scope_sha256":r.scope_hash,
"receipt_sha256":hash(&signed.payload),"request_id":r.request_id,"witness_sha256":r.witness_hash,
"approval_origin":p.approval_origin(),"testsimulation":true})).execute(&mut *tx).await?;
r.existing = Some((receipt.receipt_id, receipt.valid_until, signed));
}
tx.commit().await?;
Ok(Json(r.view()))
}
async fn download(
State(s): State<crate::State>,
Path((tenant, op)): Path<(Uuid, Uuid)>,
h: HeaderMap,
) -> Result<Json<SignedReceipt>> {
let p = access(&s, &h, tenant, false).await?;
let mut tx = rmm_persistence::tenant_tx(&s.pool, tenant).await?;
reviewer(&mut tx, &p).await?;
let r = review(&mut tx, &s, op, &case_witness())
.await?
.existing
.ok_or_else(missing)?;
tx.commit().await?;
Ok(Json(r.2))
}
#[cfg(test)]
pub(super) async fn test_fixed_echo(
admin: &sqlx::PgPool,
s: &crate::State,
mut p: Principal,
w: Value,
) {
use rmm_domain::{
jobs::{Action, JobPayload, PlannedJob, WindowsExecutionProfile},
windows_echo_probe::{OUTPUT, SCRIPT},
};
p.name = TEST_PRINCIPAL.into();
p.testsimulation = true;
p.identity = None;
let op = Uuid::new_v4();
let now = Utc::now();
let job = JobPayload {
schema_version: 1,
tenant_id: TENANT,
asset_id: ASSET,
agent_id: AGENT,
enrollment_generation: 1,
assignment_version: 1,
job_id: Uuid::new_v4(),
attempt_id: Uuid::new_v4(),
action: Action::RunScript {
interpreter: "powershell".into(),
script_utf8: SCRIPT.into(),
arguments: vec![],
expected_stdout: Some(OUTPUT.into()),
windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned),
privileged_script_profile: None,
},
execution_identity: "agent_service".into(),
not_before: now,
start_before: now + Duration::minutes(15),
max_runtime_seconds: 60,
max_output_bytes: 65536,
};
let raw = encode(&job).unwrap();
let mut plan = PlanPayload {
schema_version: 1,
tenant_id: TENANT,
plan_id: Uuid::new_v4(),
principal: format!("automation:{TENANT}"),
reason: "Echo fixture".into(),
jobs: vec![PlannedJob {
payload: job.clone(),
job_payload_hash: hash(&raw),
script_sha256: Some(hash(SCRIPT.as_bytes())),
}],
max_parallel: 1,
halt_after_failures: 1,
recipe_references: vec![],
artifact_references: vec![],
};
let scope = ProbeScope::from_plan(&plan, op).unwrap();
for field in [
"execution_identity",
"script",
"args",
"attempt",
"duplicate",
"output",
] {
let mut other = plan.clone();
let j = &mut other.jobs[0].payload;
match field {
"execution_identity" => j.execution_identity = "system".into(),
"attempt" => j.attempt_id = Uuid::new_v4(),
"output" => j.max_output_bytes += 1,
"script" | "args" => {
if let Action::RunScript {
arguments,
script_utf8,
..
} = &mut j.action
{
if field == "args" {
arguments.push("extra".into());
} else {
*script_utf8 = "Write-Output 'anderer Test'".into();
}
}
}
_ => {}
}
other.jobs[0].job_payload_hash = hash(&encode(&other.jobs[0].payload).unwrap());
if field == "duplicate" {
other.jobs.push(other.jobs[0].clone());
}
if field != "attempt" {
assert!(ProbeScope::from_plan(&other, op).is_none());
}
if field != "duplicate" {
assert!(!scope.matches_job(&other.jobs[0].payload));
}
}
let bytes = encode(&plan).unwrap();
sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,$3,$4,$5,$6)")
.bind(TENANT).bind(plan.plan_id).bind(&plan.principal).bind(&bytes).bind(&scope.plan_hash).bind(job.start_before).execute(admin).await.unwrap();
sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash) VALUES($1,$2,$3,$4,'echo','synthetic')")
.bind(TENANT).bind(op).bind(plan.plan_id).bind(&plan.principal).execute(admin).await.unwrap();
sqlx::query(
"UPDATE agent_installations SET sync_required=true,capabilities=$1 WHERE agent_id=$2",
)
.bind(json!([
"windows.base/1",
"heartbeat/1",
"windows.echo-probe/1",
"script.powershell/1",
"run_script",
"rmm.sync.v1",
"script.powershell.process_remote_signed/1"
]))
.bind(AGENT)
.execute(admin)
.await
.unwrap();
let input = || Confirmation {
reviewed: true,
witness_sha256: inventory_sha256(&w).unwrap(),
scope_sha256: scope.digest().unwrap(),
};
let mut wrong = input();
wrong.scope_sha256 = "00".repeat(32);
assert_eq!(
confirm_review(s, &p, op, wrong, &w).await.unwrap_err().1,
"WINDOWS_ECHO_SCOPE_MISMATCH"
);
let mut bearer = p.clone();
bearer.session_hash = None;
assert_eq!(
confirm_review(s, &bearer, op, input(), &w)
.await
.unwrap_err()
.0,
StatusCode::FORBIDDEN
);
let first = confirm_review(s, &p, op, input(), &w).await.unwrap().0;
assert_eq!(first["scope_sha256"], scope.digest().unwrap());
assert_eq!(
confirm_review(s, &p, op, input(), &w).await.unwrap().0,
first
);
let state:(bool,bool,bool)=sqlx::query_as("SELECT restore_blocked,sync_ready,helper_sync_ready FROM agent_installations WHERE agent_id=$1").bind(AGENT).fetch_one(admin).await.unwrap();
assert_eq!(state, (true, false, false));
let count:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs)+(SELECT count(*) FROM authorizations)+(SELECT count(*) FROM execution_permits)").fetch_one(admin).await.unwrap();
assert_eq!(count, 0);
// Even a former server binary with table-level INSERT reaches the persistent trigger.
let authorization = Uuid::new_v4();
sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,$6,$7,'dev_test_browser_simulation',true,1)")
.bind(TENANT).bind(authorization).bind(op).bind(plan.plan_id).bind(&scope.plan_hash).bind(TEST_PRINCIPAL).bind("aa".repeat(32)).execute(admin).await.unwrap();
sqlx::query("INSERT INTO operations VALUES($1,$2,$3)")
.bind(TENANT)
.bind(op)
.bind(authorization)
.execute(admin)
.await
.unwrap();
let insert = "INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'script')";
plan.jobs[0].payload.execution_identity = "system".into();
let alternate = encode(&plan.jobs[0].payload).unwrap();
let err = sqlx::query(insert)
.bind(TENANT)
.bind(job.job_id)
.bind(job.attempt_id)
.bind(op)
.bind(AGENT)
.bind(ASSET)
.bind(&alternate)
.bind(hash(&alternate))
.execute(admin)
.await
.unwrap_err();
assert!(err.to_string().contains("WINDOWS_ECHO_SCOPE_MISMATCH"));
let mut tx = rmm_persistence::tenant_tx(&s.pool, TENANT).await.unwrap();
sqlx::query(insert)
.bind(TENANT)
.bind(job.job_id)
.bind(job.attempt_id)
.bind(op)
.bind(AGENT)
.bind(ASSET)
.bind(&raw)
.bind(hash(&raw))
.execute(&mut *tx)
.await
.unwrap();
sqlx::query("INSERT INTO outbox(tenant_id,job_id,attempt_id) VALUES($1,$2,$3)")
.bind(TENANT)
.bind(job.job_id)
.bind(job.attempt_id)
.execute(&mut *tx)
.await
.unwrap();
sqlx::query("UPDATE outbox SET lease_until=now()+interval '30 seconds' WHERE job_id=$1")
.bind(job.job_id)
.execute(&mut *tx)
.await
.unwrap();
tx.commit().await.unwrap();
let grant = Uuid::new_v4();
let g = json!({"domain":"rmm.execution-permit","schema_version":1,"grant_id":grant,"authorization_id":authorization,
"tenant_id":TENANT,"asset_id":ASSET,"agent_id":AGENT,"enrollment_generation":1,"assignment_version":1,
"job_id":job.job_id,"attempt_id":job.attempt_id,"plan_hash":scope.plan_hash,"job_payload_hash":scope.job_payload_hash,
"action":"run_script","execution_identity":"agent_service","not_before":now,"start_before":now+Duration::seconds(60),
"max_runtime_seconds":60,"key_id":s.job_signer.key_id});
let insert_grant = "INSERT INTO execution_permits(tenant_id,grant_id,job_id,attempt_id,payload,signature) VALUES($1,$2,$3,$4,$5,$6)";
for valid in [false, true] {
let mut g = g.clone();
if !valid {
g["execution_identity"] = json!("system");
}
let b = serde_json::to_vec(&g).unwrap();
let signature = s.job_signer.sign(&b).unwrap();
let mut tx = rmm_persistence::tenant_tx(&s.pool, TENANT).await.unwrap();
let result = sqlx::query(insert_grant)
.bind(TENANT)
.bind(grant)
.bind(job.job_id)
.bind(job.attempt_id)
.bind(b)
.bind(signature)
.execute(&mut *tx)
.await;
if valid {
result.unwrap();
tx.commit().await.unwrap();
} else {
assert!(
result
.unwrap_err()
.to_string()
.contains("WINDOWS_ECHO_SCOPE_MISMATCH")
);
tx.rollback().await.unwrap();
}
}
crate::windows_echo_completion::test_completed_echo(admin,s,&p,&w).await;
}