//! Bind the development checkpoint intervention to a single immutable LocalService echo. //! The scope outlives its signed transport receipt and is also enforced by PostgreSQL. use crate::{ auth::Principal, jobs::Error, windows_checkpoint_review::{access, verify_probe_case}, }; use axum::{ Json, Router, extract::{Path, State}, http::{HeaderMap, StatusCode}, routing::{get, post}, }; use chrono::{DateTime, Duration, Utc}; use openssl::{ pkey::{Id, PKey}, sign::Verifier, }; use rmm_domain::{ checkpoint_repair::{SignedReceipt, inventory_sha256}, jobs::{PlanPayload, encode, hash}, windows_echo_probe::{AGENT, ASSET, DOMAIN, ProbeReceipt, ProbeScope, TENANT, TEST_PRINCIPAL}, windows_sync_history::case_witness, }; use serde::Deserialize; use serde_json::{Value, json}; use sqlx::{Postgres, Row, Transaction}; use uuid::Uuid; type Result = std::result::Result; fn mismatch() -> Error { Error(StatusCode::CONFLICT, "WINDOWS_ECHO_SCOPE_MISMATCH") } fn forbidden() -> Error { Error(StatusCode::FORBIDDEN, "DEV_TEST_WEBUI_REQUIRED") } fn missing() -> Error { Error(StatusCode::NOT_FOUND, "NOT_FOUND") } pub fn routes(r: Router) -> Router { r.route( "/api/v1/tenants/{tenant}/windows-echo-probe/{operation}", get(read), ) .route( "/api/v1/tenants/{tenant}/windows-echo-probe/{operation}/confirm", post(confirm), ) .route( "/api/v1/tenants/{tenant}/windows-echo-probe/{operation}/receipt", get(download), ) } fn browser(p: &Principal) -> Result<&str> { if p.name != TEST_PRINCIPAL || p.tenant != TENANT || !p.testsimulation || !p.operator || p.identity.is_some() || p.scopes.is_some() || p.csrf.is_none() { return Err(forbidden()); } p.session_hash.as_deref().ok_or_else(forbidden) } async fn reviewer(tx: &mut Transaction<'_, Postgres>, p: &Principal) -> Result<()> { browser(p)?; let allowed: bool = sqlx::query_scalar("SELECT lock_windows_echo_probe_reviewer()") .fetch_one(&mut **tx) .await?; if !allowed { return Err(forbidden()); } Ok(()) } struct Review { scope: ProbeScope, raw_job: Vec, request_id: Uuid, witness: Value, witness_hash: String, scope_hash: String, existing: Option<(Uuid, DateTime, SignedReceipt)>, } impl Review { fn view(&self) -> Value { json!({"scope":self.scope,"scope_sha256":self.scope_hash,"witness_sha256":self.witness_hash,"witness":self.witness, "receipt_id":self.existing.as_ref().map(|r|r.0),"valid_until":self.existing.as_ref().map(|r|r.1)}) } } async fn review( tx: &mut Transaction<'_, Postgres>, s: &crate::State, op: Uuid, expected: &Value, ) -> Result { // Same control lock ordering as jobs and checkpoint recovery. crate::agent_service::control_lock(tx, TENANT, AGENT).await?; let r = sqlx::query( "SELECT o.payload,o.plan_hash,a.approval_state,w.request_id,w.witness,w.witness_sha256 FROM approval_requests a JOIN operation_plans o USING(tenant_id,plan_id) JOIN windows_checkpoint_reviews w ON w.tenant_id=a.tenant_id AND w.agent_id=$3 WHERE a.tenant_id=$1 AND a.operation_id=$2 FOR UPDATE OF a", ) .bind(TENANT) .bind(op) .bind(AGENT) .fetch_optional(&mut **tx) .await? .ok_or_else(missing)?; if r.get::("approval_state") != "pending_approval" { return Err(mismatch()); } let plan_bytes: Vec = r.get("payload"); let plan: PlanPayload = serde_json::from_slice(&plan_bytes)?; let scope = ProbeScope::from_plan(&plan, op).ok_or_else(mismatch)?; if scope.plan_hash != r.get::("plan_hash") || hash(&plan_bytes) != scope.plan_hash || scope.start_before <= Utc::now() { return Err(mismatch()); } let witness: Value = r.get("witness"); let witness_hash = inventory_sha256(&witness)?; if &witness != expected || witness_hash != r.get::("witness_sha256") { return Err(mismatch()); } let scope_hash = scope.digest().ok_or_else(mismatch)?; let stored=sqlx::query("SELECT operation_id,receipt_id,valid_until,receipt,signature FROM windows_echo_probe_scopes WHERE tenant_id=$1") .bind(TENANT).fetch_optional(&mut **tx).await?; let mut existing = None; if let Some(old) = stored { if old.get::("operation_id") != op { return Err(mismatch()); } let signed = SignedReceipt { payload: old.get("receipt"), signature: old.get("signature"), }; let receipt: ProbeReceipt = serde_json::from_slice(&signed.payload)?; let key = PKey::public_key_from_raw_bytes(&s.job_signer.public_key()?, Id::ED25519) .map_err(|_| mismatch())?; if !Verifier::new_without_digest(&key) .map_err(|_| mismatch())? .verify_oneshot(&signed.signature, &signed.payload) .map_err(|_| mismatch())? || !receipt.matches_reviewed_witness(expected, Utc::now()) || receipt.scope != scope || receipt.receipt_id != old.get::("receipt_id") || receipt.request_id != r.get::("request_id") || receipt.valid_until.timestamp_micros() != old .get::, _>("valid_until") .timestamp_micros() { return Err(mismatch()); } existing = Some((receipt.receipt_id, receipt.valid_until, signed)); } verify_probe_case(tx, s, &witness, existing.is_some()).await?; Ok(Review { raw_job: encode(&plan.jobs[0].payload)?, scope, scope_hash, request_id: r.get("request_id"), witness, witness_hash, existing, }) } async fn read( State(s): State, Path((tenant, op)): Path<(Uuid, Uuid)>, h: HeaderMap, ) -> Result> { let p = access(&s, &h, tenant, false).await?; let mut tx = rmm_persistence::tenant_tx(&s.pool, tenant).await?; crate::permissions::require_principal(&mut tx, &p, "operations:read").await?; let info = review(&mut tx, &s, op, &case_witness()).await?.view(); tx.commit().await?; Ok(Json(info)) } #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct Confirmation { reviewed: bool, scope_sha256: String, witness_sha256: String, } async fn confirm( State(s): State, Path((tenant, op)): Path<(Uuid, Uuid)>, h: HeaderMap, Json(input): Json, ) -> Result> { let p = access(&s, &h, tenant, true).await?; confirm_review(&s, &p, op, input, &case_witness()).await } async fn confirm_review( s: &crate::State, p: &Principal, op: Uuid, input: Confirmation, expected: &Value, ) -> Result> { let session = browser(p)?; if !input.reviewed { return Err(Error(StatusCode::BAD_REQUEST, "REVIEW_REQUIRED")); } let mut tx = rmm_persistence::tenant_tx(&s.pool, p.tenant).await?; reviewer(&mut tx, p).await?; let mut r = review(&mut tx, s, op, expected).await?; if input.scope_sha256 != r.scope_hash || input.witness_sha256 != r.witness_hash { return Err(mismatch()); } if r.existing.is_none() { let now = Utc::now(); let receipt = ProbeReceipt { domain: DOMAIN.into(), schema_version: 1, request_id: r.request_id, receipt_id: Uuid::new_v4(), witness: r.witness.clone(), witness_sha256: r.witness_hash.clone(), scope: r.scope.clone(), scope_sha256: r.scope_hash.clone(), job_payload_bytes: r.raw_job.clone(), key_id: s.job_signer.key_id.clone(), issued_at: now, valid_until: std::cmp::min(now + Duration::minutes(10), r.scope.start_before), confirmed_by: p.name.clone(), approval_origin: p.approval_origin().into(), testsimulation: true, }; if !receipt.matches_reviewed_witness(expected, now) { return Err(mismatch()); } let raw = serde_json::to_vec(&receipt)?; let signed = SignedReceipt { signature: s.job_signer.sign(&raw)?, payload: raw, }; sqlx::query("INSERT INTO windows_echo_probe_scopes(tenant_id,asset_id,agent_id,operation_id,plan_id,plan_hash,job_id,attempt_id,job_bytes,job_hash,request_id,receipt_id,receipt,signature,confirmed_by,session_hash,issued_at,valid_until) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)") .bind(TENANT).bind(ASSET).bind(AGENT).bind(op).bind(r.scope.plan_id).bind(&r.scope.plan_hash).bind(r.scope.job_id).bind(r.scope.attempt_id) .bind(&r.raw_job).bind(&r.scope.job_payload_hash).bind(r.request_id).bind(receipt.receipt_id).bind(&signed.payload).bind(&signed.signature) .bind(&p.name).bind(session).bind(now).bind(receipt.valid_until).execute(&mut *tx).await?; // Only after installing the immutable scope in the same transaction. if sqlx::query("UPDATE agent_sync_state SET checkpoint=6756,decision_id=$4 WHERE tenant_id=$1 AND agent_id=$2 AND checkpoint=6755 AND decision_id=$3 AND helper_checkpoint=1") .bind(TENANT).bind(AGENT).bind(crate::windows_checkpoint_review::FIRST).bind(crate::windows_checkpoint_review::DENIED) .execute(&mut *tx).await?.rows_affected()!=1{return Err(mismatch());} sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,'windows_echo_probe_scoped',$4)") .bind(TENANT).bind(receipt.receipt_id).bind(&p.name).bind(json!({"scope":r.scope,"scope_sha256":r.scope_hash, "receipt_sha256":hash(&signed.payload),"request_id":r.request_id,"witness_sha256":r.witness_hash, "approval_origin":p.approval_origin(),"testsimulation":true})).execute(&mut *tx).await?; r.existing = Some((receipt.receipt_id, receipt.valid_until, signed)); } tx.commit().await?; Ok(Json(r.view())) } async fn download( State(s): State, Path((tenant, op)): Path<(Uuid, Uuid)>, h: HeaderMap, ) -> Result> { let p = access(&s, &h, tenant, false).await?; let mut tx = rmm_persistence::tenant_tx(&s.pool, tenant).await?; reviewer(&mut tx, &p).await?; let r = review(&mut tx, &s, op, &case_witness()) .await? .existing .ok_or_else(missing)?; tx.commit().await?; Ok(Json(r.2)) } #[cfg(test)] pub(super) async fn test_fixed_echo( admin: &sqlx::PgPool, s: &crate::State, mut p: Principal, w: Value, ) { use rmm_domain::{ jobs::{Action, JobPayload, PlannedJob, WindowsExecutionProfile}, windows_echo_probe::{OUTPUT, SCRIPT}, }; p.name = TEST_PRINCIPAL.into(); p.testsimulation = true; p.identity = None; let op = Uuid::new_v4(); let now = Utc::now(); let job = JobPayload { schema_version: 1, tenant_id: TENANT, asset_id: ASSET, agent_id: AGENT, enrollment_generation: 1, assignment_version: 1, job_id: Uuid::new_v4(), attempt_id: Uuid::new_v4(), action: Action::RunScript { interpreter: "powershell".into(), script_utf8: SCRIPT.into(), arguments: vec![], expected_stdout: Some(OUTPUT.into()), windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned), privileged_script_profile: None, }, execution_identity: "agent_service".into(), not_before: now, start_before: now + Duration::minutes(15), max_runtime_seconds: 60, max_output_bytes: 65536, }; let raw = encode(&job).unwrap(); let mut plan = PlanPayload { schema_version: 1, tenant_id: TENANT, plan_id: Uuid::new_v4(), principal: format!("automation:{TENANT}"), reason: "Echo fixture".into(), jobs: vec![PlannedJob { payload: job.clone(), job_payload_hash: hash(&raw), script_sha256: Some(hash(SCRIPT.as_bytes())), }], max_parallel: 1, halt_after_failures: 1, recipe_references: vec![], artifact_references: vec![], }; let scope = ProbeScope::from_plan(&plan, op).unwrap(); for field in [ "execution_identity", "script", "args", "attempt", "duplicate", "output", ] { let mut other = plan.clone(); let j = &mut other.jobs[0].payload; match field { "execution_identity" => j.execution_identity = "system".into(), "attempt" => j.attempt_id = Uuid::new_v4(), "output" => j.max_output_bytes += 1, "script" | "args" => { if let Action::RunScript { arguments, script_utf8, .. } = &mut j.action { if field == "args" { arguments.push("extra".into()); } else { *script_utf8 = "Write-Output 'anderer Test'".into(); } } } _ => {} } other.jobs[0].job_payload_hash = hash(&encode(&other.jobs[0].payload).unwrap()); if field == "duplicate" { other.jobs.push(other.jobs[0].clone()); } if field != "attempt" { assert!(ProbeScope::from_plan(&other, op).is_none()); } if field != "duplicate" { assert!(!scope.matches_job(&other.jobs[0].payload)); } } let bytes = encode(&plan).unwrap(); sqlx::query("INSERT INTO operation_plans(tenant_id,plan_id,principal,payload,plan_hash,expires_at) VALUES($1,$2,$3,$4,$5,$6)") .bind(TENANT).bind(plan.plan_id).bind(&plan.principal).bind(&bytes).bind(&scope.plan_hash).bind(job.start_before).execute(admin).await.unwrap(); sqlx::query("INSERT INTO approval_requests(tenant_id,operation_id,plan_id,principal,idempotency_key,request_hash) VALUES($1,$2,$3,$4,'echo','synthetic')") .bind(TENANT).bind(op).bind(plan.plan_id).bind(&plan.principal).execute(admin).await.unwrap(); sqlx::query( "UPDATE agent_installations SET sync_required=true,capabilities=$1 WHERE agent_id=$2", ) .bind(json!([ "windows.base/1", "heartbeat/1", "windows.echo-probe/1", "script.powershell/1", "run_script", "rmm.sync.v1", "script.powershell.process_remote_signed/1" ])) .bind(AGENT) .execute(admin) .await .unwrap(); let input = || Confirmation { reviewed: true, witness_sha256: inventory_sha256(&w).unwrap(), scope_sha256: scope.digest().unwrap(), }; let mut wrong = input(); wrong.scope_sha256 = "00".repeat(32); assert_eq!( confirm_review(s, &p, op, wrong, &w).await.unwrap_err().1, "WINDOWS_ECHO_SCOPE_MISMATCH" ); let mut bearer = p.clone(); bearer.session_hash = None; assert_eq!( confirm_review(s, &bearer, op, input(), &w) .await .unwrap_err() .0, StatusCode::FORBIDDEN ); let first = confirm_review(s, &p, op, input(), &w).await.unwrap().0; assert_eq!(first["scope_sha256"], scope.digest().unwrap()); assert_eq!( confirm_review(s, &p, op, input(), &w).await.unwrap().0, first ); let state:(bool,bool,bool)=sqlx::query_as("SELECT restore_blocked,sync_ready,helper_sync_ready FROM agent_installations WHERE agent_id=$1").bind(AGENT).fetch_one(admin).await.unwrap(); assert_eq!(state, (true, false, false)); let count:i64=sqlx::query_scalar("SELECT (SELECT count(*) FROM jobs)+(SELECT count(*) FROM authorizations)+(SELECT count(*) FROM execution_permits)").fetch_one(admin).await.unwrap(); assert_eq!(count, 0); // Even a former server binary with table-level INSERT reaches the persistent trigger. let authorization = Uuid::new_v4(); sqlx::query("INSERT INTO authorizations(tenant_id,authorization_id,operation_id,plan_id,plan_hash,principal,session_hash,approval_origin,testsimulation,policy_version) VALUES($1,$2,$3,$4,$5,$6,$7,'dev_test_browser_simulation',true,1)") .bind(TENANT).bind(authorization).bind(op).bind(plan.plan_id).bind(&scope.plan_hash).bind(TEST_PRINCIPAL).bind("aa".repeat(32)).execute(admin).await.unwrap(); sqlx::query("INSERT INTO operations VALUES($1,$2,$3)") .bind(TENANT) .bind(op) .bind(authorization) .execute(admin) .await .unwrap(); let insert = "INSERT INTO jobs(tenant_id,job_id,attempt_id,operation_id,agent_id,asset_id,payload,payload_hash,resource) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'script')"; plan.jobs[0].payload.execution_identity = "system".into(); let alternate = encode(&plan.jobs[0].payload).unwrap(); let err = sqlx::query(insert) .bind(TENANT) .bind(job.job_id) .bind(job.attempt_id) .bind(op) .bind(AGENT) .bind(ASSET) .bind(&alternate) .bind(hash(&alternate)) .execute(admin) .await .unwrap_err(); assert!(err.to_string().contains("WINDOWS_ECHO_SCOPE_MISMATCH")); let mut tx = rmm_persistence::tenant_tx(&s.pool, TENANT).await.unwrap(); sqlx::query(insert) .bind(TENANT) .bind(job.job_id) .bind(job.attempt_id) .bind(op) .bind(AGENT) .bind(ASSET) .bind(&raw) .bind(hash(&raw)) .execute(&mut *tx) .await .unwrap(); sqlx::query("INSERT INTO outbox(tenant_id,job_id,attempt_id) VALUES($1,$2,$3)") .bind(TENANT) .bind(job.job_id) .bind(job.attempt_id) .execute(&mut *tx) .await .unwrap(); sqlx::query("UPDATE outbox SET lease_until=now()+interval '30 seconds' WHERE job_id=$1") .bind(job.job_id) .execute(&mut *tx) .await .unwrap(); tx.commit().await.unwrap(); let grant = Uuid::new_v4(); let g = json!({"domain":"rmm.execution-permit","schema_version":1,"grant_id":grant,"authorization_id":authorization, "tenant_id":TENANT,"asset_id":ASSET,"agent_id":AGENT,"enrollment_generation":1,"assignment_version":1, "job_id":job.job_id,"attempt_id":job.attempt_id,"plan_hash":scope.plan_hash,"job_payload_hash":scope.job_payload_hash, "action":"run_script","execution_identity":"agent_service","not_before":now,"start_before":now+Duration::seconds(60), "max_runtime_seconds":60,"key_id":s.job_signer.key_id}); let insert_grant = "INSERT INTO execution_permits(tenant_id,grant_id,job_id,attempt_id,payload,signature) VALUES($1,$2,$3,$4,$5,$6)"; for valid in [false, true] { let mut g = g.clone(); if !valid { g["execution_identity"] = json!("system"); } let b = serde_json::to_vec(&g).unwrap(); let signature = s.job_signer.sign(&b).unwrap(); let mut tx = rmm_persistence::tenant_tx(&s.pool, TENANT).await.unwrap(); let result = sqlx::query(insert_grant) .bind(TENANT) .bind(grant) .bind(job.job_id) .bind(job.attempt_id) .bind(b) .bind(signature) .execute(&mut *tx) .await; if valid { result.unwrap(); tx.commit().await.unwrap(); } else { assert!( result .unwrap_err() .to_string() .contains("WINDOWS_ECHO_SCOPE_MISMATCH") ); tx.rollback().await.unwrap(); } } crate::windows_echo_completion::test_completed_echo(admin,s,&p,&w).await; }