Add reviewed Windows scope completion and idle certificate renewal; deploy completion API
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-16 10:25:25 +01:00
parent ebcad08731
commit bb4ffe518d
43 changed files with 5146 additions and 50 deletions
@@ -0,0 +1,444 @@
//! Interactive retirement of the temporary fixed-probe restriction. Never approves jobs.
use crate::{auth::Principal, jobs::Error, windows_checkpoint_review::access};
use axum::{
Json, Router,
extract::{Path, State},
http::{HeaderMap, StatusCode},
routing::{get, post},
};
use chrono::{Duration, SubsecRound, Utc};
use openssl::{
pkey::{Id, PKey},
sign::Verifier,
};
use rmm_domain::{
checkpoint_repair::{SignedReceipt, inventory_sha256},
jobs::{ExecutionPermitPayload, JobPayload, hash},
windows_echo_completion::{DOMAIN, Receipt, Witness, successful_outcome},
windows_echo_probe::{AGENT, ASSET, ProbeReceipt, TENANT, TEST_PRINCIPAL},
windows_sync_history::case_witness,
};
use serde::Deserialize;
use serde_json::{Value, json};
use sqlx::{Postgres, Row, Transaction};
use uuid::Uuid;
type Result<T> = std::result::Result<T, Error>;
fn mismatch() -> Error {
Error(StatusCode::CONFLICT, "WINDOWS_ECHO_COMPLETION_MISMATCH")
}
fn browser(p: &Principal) -> Result<&str> {
if p.name != TEST_PRINCIPAL
|| p.tenant != TENANT
|| !p.testsimulation
|| !p.operator
|| p.identity.is_some()
|| p.scopes.is_some()
|| p.csrf.is_none()
{
return Err(Error(StatusCode::FORBIDDEN, "DEV_TEST_WEBUI_REQUIRED"));
}
p.session_hash
.as_deref()
.ok_or(Error(StatusCode::FORBIDDEN, "DEV_TEST_WEBUI_REQUIRED"))
}
pub fn routes(r: Router<crate::State>) -> Router<crate::State> {
r.route(
"/api/v1/tenants/{tenant}/windows-echo-completion",
get(read),
)
.route(
"/api/v1/tenants/{tenant}/windows-echo-completion/confirm",
post(confirm),
)
.route(
"/api/v1/tenants/{tenant}/windows-echo-completion/receipt",
get(download),
)
}
fn verify(s: &crate::State, bytes: &[u8], signature: &[u8]) -> Result<()> {
let key = PKey::public_key_from_raw_bytes(&s.job_signer.public_key()?, Id::ED25519)
.map_err(|_| mismatch())?;
if bytes.len() > 65536
|| signature.len() != 64
|| !Verifier::new_without_digest(&key)
.map_err(|_| mismatch())?
.verify_oneshot(signature, bytes)
.map_err(|_| mismatch())?
{
return Err(mismatch());
}
Ok(())
}
struct Review {
witness: Witness,
existing: Option<SignedReceipt>,
}
impl Review {
fn view(&self) -> Value {
json!({"witness":self.witness,"witness_sha256":self.witness.digest(),
"receipt":self.existing.as_ref().and_then(|s|serde_json::from_slice::<Value>(&s.payload).ok())})
}
}
async fn review(tx: &mut Transaction<'_, Postgres>, s: &crate::State) -> Result<Review> {
review_for(tx, s, &case_witness()).await
}
async fn review_for(
tx: &mut Transaction<'_, Postgres>,
s: &crate::State,
expected: &Value,
) -> Result<Review> {
crate::agent_service::control_lock(tx, TENANT, AGENT).await?;
let dev=sqlx::query("SELECT asset_id,revoked_at IS NULL AS active,enrollment_generation,assignment_version,connection_boot_id,restore_blocked,sequence_regressed,sync_recovery_required FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
.bind(TENANT).bind(AGENT).fetch_one(&mut **tx).await?;
if dev.get::<Uuid, _>("asset_id") != ASSET
|| !dev.get::<bool, _>("active")
|| dev.get::<i64, _>("enrollment_generation") != 1
|| dev.get::<i64, _>("assignment_version") != 1
|| dev.get::<bool, _>("restore_blocked")
|| dev.get::<bool, _>("sequence_regressed")
|| dev.get::<bool, _>("sync_recovery_required")
{
return Err(mismatch());
}
let row = sqlx::query(
"SELECT receipt,signature,receipt_id FROM windows_echo_probe_scopes WHERE tenant_id=$1",
)
.bind(TENANT)
.fetch_one(&mut **tx)
.await?;
let raw: Vec<u8> = row.get("receipt");
let sig: Vec<u8> = row.get("signature");
verify(s, &raw, &sig)?;
let probe: ProbeReceipt = serde_json::from_slice(&raw)?;
if !probe.matches_reviewed_witness(expected, probe.issued_at)
|| probe.receipt_id != row.get::<Uuid, _>("receipt_id")
{
return Err(mismatch());
}
let scope = &probe.scope;
let job=sqlx::query("SELECT state,payload,payload_hash,result FROM jobs WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND job_id=$4 AND attempt_id=$5 AND operation_id=$6 FOR SHARE")
.bind(TENANT).bind(AGENT).bind(ASSET).bind(scope.job_id).bind(scope.attempt_id).bind(scope.operation_id).fetch_one(&mut **tx).await?;
let mut outcome: Value = job.get::<Option<Value>, _>("result").ok_or_else(mismatch)?;
outcome
.as_object_mut()
.ok_or_else(mismatch)?
.remove("grant_id");
outcome["state"] = json!(job.get::<String, _>("state"));
let job_bytes: Vec<u8> = job.get("payload");
let payload: JobPayload = serde_json::from_slice(&job_bytes)?;
if job_bytes != probe.job_payload_bytes
|| !scope.matches_job(&payload)
|| job.get::<String, _>("payload_hash") != scope.job_payload_hash
|| !successful_outcome(&outcome)
{
return Err(mismatch());
}
let extra: bool = sqlx::query_scalar(
"SELECT EXISTS(SELECT FROM jobs WHERE tenant_id=$1 AND agent_id=$2 AND job_id!=$3)",
)
.bind(TENANT)
.bind(AGENT)
.bind(scope.job_id)
.fetch_one(&mut **tx)
.await?;
let existing =
sqlx::query("SELECT payload,signature FROM windows_echo_completions WHERE tenant_id=$1")
.bind(TENANT)
.fetch_optional(&mut **tx)
.await?;
// Only the first transition requires the original single-job inventory.
if extra && existing.is_none() {
return Err(mismatch());
}
let grant=sqlx::query("SELECT grant_id,payload,signature FROM execution_permits WHERE tenant_id=$1 AND job_id=$2 AND attempt_id=$3")
.bind(TENANT).bind(scope.job_id).bind(scope.attempt_id).fetch_one(&mut **tx).await?;
let graw: Vec<u8> = grant.get("payload");
let gsig: Vec<u8> = grant.get("signature");
verify(s, &graw, &gsig)?;
let g: ExecutionPermitPayload = serde_json::from_slice(&graw)?;
if g.domain != "rmm.execution-permit"
|| g.schema_version != 1
|| g.action != "run_script"
|| g.agent_update_plan_id.is_some()
|| (g.enrollment_generation, g.assignment_version)
!= (scope.enrollment_generation, scope.assignment_version)
|| g.max_runtime_seconds != scope.max_runtime_seconds
|| g.not_before < scope.not_before
|| g.start_before > scope.start_before
|| g.not_before >= g.start_before
|| g.start_before - g.not_before > Duration::seconds(60)
|| g.grant_id != grant.get::<Uuid, _>("grant_id")
|| g.job_id != scope.job_id
|| g.attempt_id != scope.attempt_id
|| g.plan_hash != scope.plan_hash
|| g.job_payload_hash != scope.job_payload_hash
|| g.agent_id != AGENT
|| g.tenant_id != TENANT
|| g.asset_id != ASSET
|| g.execution_identity != "agent_service"
|| g.key_id != s.job_signer.key_id
|| job.get::<Option<Value>, _>("result").unwrap_or_default()["grant_id"]
!= g.grant_id.to_string()
{
return Err(mismatch());
}
let authorized:bool=sqlx::query_scalar("SELECT EXISTS(SELECT FROM authorizations WHERE tenant_id=$1 AND authorization_id=$2 AND operation_id=$3 AND plan_id=$4 AND plan_hash=$5 AND principal=$6 AND approval_origin='dev_test_browser_simulation' AND testsimulation AND revoked_at IS NULL)")
.bind(TENANT).bind(g.authorization_id).bind(scope.operation_id).bind(scope.plan_id).bind(&scope.plan_hash).bind(TEST_PRINCIPAL).fetch_one(&mut **tx).await?;
if !authorized {
return Err(mismatch());
}
let boot: Uuid = serde_json::from_value(expected["boot_id"].clone())?;
if dev.get::<Option<Uuid>, _>("connection_boot_id") != Some(boot) {
return Err(mismatch());
}
let witness = Witness {
scope: scope.clone(),
probe_receipt_id: probe.receipt_id,
probe_receipt_sha256: hash(&raw),
probe_signature_sha256: hash(&sig),
grant_id: g.grant_id,
grant_payload_sha256: hash(&graw),
grant_signature_sha256: hash(&gsig),
outcome_sha256: inventory_sha256(&outcome)?,
boot_id: boot,
};
let existing = existing.map(|r| SignedReceipt {
payload: r.get("payload"),
signature: r.get("signature"),
});
if let Some(signed) = &existing {
verify(s, &signed.payload, &signed.signature)?;
let r: Receipt = serde_json::from_slice(&signed.payload)?;
if !r.valid(r.issued_at) || r.witness != witness || r.key_id != s.job_signer.key_id {
return Err(mismatch());
}
}
Ok(Review { witness, existing })
}
async fn read(
State(s): State<crate::State>,
Path(t): Path<Uuid>,
h: HeaderMap,
) -> Result<Json<Value>> {
let p = access(&s, &h, t, false).await?;
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
crate::permissions::require_principal(&mut tx, &p, "operations:read").await?;
let result = review(&mut tx, &s).await?.view();
tx.commit().await?;
Ok(Json(result))
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Confirmation {
reviewed: bool,
witness_sha256: String,
}
async fn confirm(
State(s): State<crate::State>,
Path(t): Path<Uuid>,
h: HeaderMap,
Json(input): Json<Confirmation>,
) -> Result<Json<Value>> {
let p = access(&s, &h, t, true).await?;
confirm_review(&s, &p, input, &case_witness()).await
}
async fn confirm_review(
s: &crate::State,
p: &Principal,
input: Confirmation,
expected: &Value,
) -> Result<Json<Value>> {
let t = p.tenant;
let session = browser(p)?;
if !input.reviewed {
return Err(Error(StatusCode::BAD_REQUEST, "REVIEW_REQUIRED"));
}
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
let allowed: bool = sqlx::query_scalar("SELECT lock_windows_echo_probe_reviewer()")
.fetch_one(&mut *tx)
.await?;
if !allowed {
return Err(Error(StatusCode::FORBIDDEN, "DEV_TEST_WEBUI_REQUIRED"));
}
let mut r = review_for(&mut tx, s, expected).await?;
if r.witness.digest().as_deref() != Some(input.witness_sha256.as_str()) {
return Err(mismatch());
}
if r.existing.is_none() {
// SQLx binds PostgreSQL timestamps at microsecond precision. Normalize
// before signing so JSON's parser cannot round a nanosecond differently.
let now = Utc::now().trunc_subsecs(6);
let receipt = Receipt {
domain: DOMAIN.into(),
schema_version: 1,
receipt_id: Uuid::new_v4(),
witness: r.witness.clone(),
witness_sha256: input.witness_sha256,
key_id: s.job_signer.key_id.clone(),
issued_at: now,
valid_until: now + Duration::minutes(10),
confirmed_by: p.name.clone(),
approval_origin: p.approval_origin().into(),
testsimulation: true,
ready: false,
helper_ready: false,
unlock: false,
};
if !receipt.valid(now) {
return Err(mismatch());
}
let raw = serde_json::to_vec(&receipt)?;
let signature = s.job_signer.sign(&raw)?;
sqlx::query("INSERT INTO windows_echo_completions(tenant_id,receipt_id,witness_sha256,payload,signature,confirmed_by,session_hash,issued_at,valid_until) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)")
.bind(t).bind(receipt.receipt_id).bind(&receipt.witness_sha256).bind(&raw).bind(&signature).bind(&p.name).bind(session).bind(now).bind(receipt.valid_until).execute(&mut *tx).await?;
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,'windows_echo_scope_completed',$4)")
.bind(t).bind(receipt.receipt_id).bind(&p.name).bind(json!({"witness":receipt.witness,"witness_sha256":receipt.witness_sha256,
"receipt_sha256":hash(&raw),"approval_origin":receipt.approval_origin,"testsimulation":true,"ready":false,"helper_ready":false,"unlock":false})).execute(&mut *tx).await?;
r.existing = Some(SignedReceipt {
payload: raw,
signature,
});
}
tx.commit().await?;
Ok(Json(r.view()))
}
async fn download(
State(s): State<crate::State>,
Path(t): Path<Uuid>,
h: HeaderMap,
) -> Result<Json<SignedReceipt>> {
let p = access(&s, &h, t, false).await?;
browser(&p)?;
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
crate::permissions::require_principal(&mut tx, &p, "plans:approve").await?;
let signed = review(&mut tx, &s)
.await?
.existing
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?;
let r: Receipt = serde_json::from_slice(&signed.payload)?;
if !r.valid(Utc::now()) {
return Err(Error(
StatusCode::CONFLICT,
"WINDOWS_ECHO_COMPLETION_EXPIRED",
));
}
tx.commit().await?;
Ok(Json(signed))
}
#[cfg(test)]
pub(super) async fn test_completed_echo(
admin: &sqlx::PgPool,
s: &crate::State,
p: &Principal,
expected: &Value,
) {
let nanosecond = chrono::DateTime::from_timestamp(1794820000, 123456789).unwrap();
let compare = |t: chrono::DateTime<Utc>| {
sqlx::query_scalar::<_, bool>("SELECT $1::timestamptz=$2::text::timestamptz")
.bind(t)
.bind(t.to_rfc3339())
};
assert!(!compare(nanosecond).fetch_one(admin).await.unwrap());
assert!(
compare(nanosecond.trunc_subsecs(6))
.fetch_one(admin)
.await
.unwrap()
);
let row =
sqlx::query("SELECT job_id,attempt_id,grant_id FROM execution_permits WHERE tenant_id=$1")
.bind(TENANT)
.fetch_one(admin)
.await
.unwrap();
let grant: Uuid = row.get("grant_id");
let job: Uuid = row.get("job_id");
let original_counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM jobs),(SELECT count(*) FROM authorizations),(SELECT count(*) FROM execution_permits)").fetch_one(admin).await.unwrap();
let confirm = |digest: String| Confirmation {
reviewed: true,
witness_sha256: digest,
};
assert!(
confirm_review(s, p, confirm("0".repeat(64)), expected)
.await
.is_err()
);
let mut no_browser = p.clone();
no_browser.session_hash = None;
assert_eq!(
confirm_review(s, &no_browser, confirm("0".repeat(64)), expected)
.await
.unwrap_err()
.0,
StatusCode::FORBIDDEN
);
let outcome = json!({"exit_code":0,"stdout":rmm_domain::windows_echo_probe::OUTPUT.as_bytes(),"stderr":[],"error_code":"","output_truncated":false,"reboot_required":false,
"evidence":{"execution_sid":"S-1-5-19","execution_policy_scope":"process","execution_policy_requested":"RemoteSigned","postconditions_passed":true,
"script_sha256":hash(rmm_domain::windows_echo_probe::SCRIPT.as_bytes())},"grant_id":grant});
sqlx::query("UPDATE jobs SET state='succeeded',result=$2 WHERE job_id=$1")
.bind(job)
.bind(outcome)
.execute(admin)
.await
.unwrap();
sqlx::query("UPDATE agent_installations SET restore_blocked=false WHERE agent_id=$1")
.bind(AGENT)
.execute(admin)
.await
.unwrap();
let mut tx = rmm_persistence::tenant_tx(&s.pool, TENANT).await.unwrap();
let r = review_for(&mut tx, s, expected).await.unwrap();
tx.rollback().await.unwrap();
let digest = r.witness.digest().unwrap();
assert!(
confirm_review(s, p, confirm("0".repeat(64)), expected)
.await
.is_err()
);
let first = confirm_review(s, p, confirm(digest.clone()), expected)
.await
.unwrap()
.0;
assert_eq!(first["receipt"]["ready"], false);
assert_eq!(first["receipt"]["helper_ready"], false);
assert_eq!(first["receipt"]["unlock"], false);
assert_eq!(first["receipt"]["testsimulation"], true);
assert_eq!(
confirm_review(s, p, confirm(digest), expected)
.await
.unwrap()
.0,
first
);
let counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM jobs),(SELECT count(*) FROM authorizations),(SELECT count(*) FROM execution_permits)").fetch_one(admin).await.unwrap();
assert_eq!(counts, original_counts);
assert!(
sqlx::query("DELETE FROM windows_echo_completions")
.execute(admin)
.await
.is_err()
);
assert!(
sqlx::query("UPDATE windows_echo_completions SET witness_sha256=repeat('0',64)")
.execute(admin)
.await
.is_err()
);
assert_eq!(
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM windows_echo_probe_scopes")
.fetch_one(admin)
.await
.unwrap(),
1
);
assert_eq!(
sqlx::query_scalar::<_, i64>(
"SELECT count(*) FROM audit_events WHERE event_type='windows_echo_scope_completed'"
)
.fetch_one(admin)
.await
.unwrap(),
1
);
}