Add reviewed Windows scope completion and idle certificate renewal; deploy completion API
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,444 @@
|
||||
//! Interactive retirement of the temporary fixed-probe restriction. Never approves jobs.
|
||||
use crate::{auth::Principal, jobs::Error, windows_checkpoint_review::access};
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, State},
|
||||
http::{HeaderMap, StatusCode},
|
||||
routing::{get, post},
|
||||
};
|
||||
use chrono::{Duration, SubsecRound, Utc};
|
||||
use openssl::{
|
||||
pkey::{Id, PKey},
|
||||
sign::Verifier,
|
||||
};
|
||||
use rmm_domain::{
|
||||
checkpoint_repair::{SignedReceipt, inventory_sha256},
|
||||
jobs::{ExecutionPermitPayload, JobPayload, hash},
|
||||
windows_echo_completion::{DOMAIN, Receipt, Witness, successful_outcome},
|
||||
windows_echo_probe::{AGENT, ASSET, ProbeReceipt, TENANT, TEST_PRINCIPAL},
|
||||
windows_sync_history::case_witness,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use sqlx::{Postgres, Row, Transaction};
|
||||
use uuid::Uuid;
|
||||
type Result<T> = std::result::Result<T, Error>;
|
||||
fn mismatch() -> Error {
|
||||
Error(StatusCode::CONFLICT, "WINDOWS_ECHO_COMPLETION_MISMATCH")
|
||||
}
|
||||
fn browser(p: &Principal) -> Result<&str> {
|
||||
if p.name != TEST_PRINCIPAL
|
||||
|| p.tenant != TENANT
|
||||
|| !p.testsimulation
|
||||
|| !p.operator
|
||||
|| p.identity.is_some()
|
||||
|| p.scopes.is_some()
|
||||
|| p.csrf.is_none()
|
||||
{
|
||||
return Err(Error(StatusCode::FORBIDDEN, "DEV_TEST_WEBUI_REQUIRED"));
|
||||
}
|
||||
p.session_hash
|
||||
.as_deref()
|
||||
.ok_or(Error(StatusCode::FORBIDDEN, "DEV_TEST_WEBUI_REQUIRED"))
|
||||
}
|
||||
pub fn routes(r: Router<crate::State>) -> Router<crate::State> {
|
||||
r.route(
|
||||
"/api/v1/tenants/{tenant}/windows-echo-completion",
|
||||
get(read),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/tenants/{tenant}/windows-echo-completion/confirm",
|
||||
post(confirm),
|
||||
)
|
||||
.route(
|
||||
"/api/v1/tenants/{tenant}/windows-echo-completion/receipt",
|
||||
get(download),
|
||||
)
|
||||
}
|
||||
fn verify(s: &crate::State, bytes: &[u8], signature: &[u8]) -> Result<()> {
|
||||
let key = PKey::public_key_from_raw_bytes(&s.job_signer.public_key()?, Id::ED25519)
|
||||
.map_err(|_| mismatch())?;
|
||||
if bytes.len() > 65536
|
||||
|| signature.len() != 64
|
||||
|| !Verifier::new_without_digest(&key)
|
||||
.map_err(|_| mismatch())?
|
||||
.verify_oneshot(signature, bytes)
|
||||
.map_err(|_| mismatch())?
|
||||
{
|
||||
return Err(mismatch());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
struct Review {
|
||||
witness: Witness,
|
||||
existing: Option<SignedReceipt>,
|
||||
}
|
||||
impl Review {
|
||||
fn view(&self) -> Value {
|
||||
json!({"witness":self.witness,"witness_sha256":self.witness.digest(),
|
||||
"receipt":self.existing.as_ref().and_then(|s|serde_json::from_slice::<Value>(&s.payload).ok())})
|
||||
}
|
||||
}
|
||||
async fn review(tx: &mut Transaction<'_, Postgres>, s: &crate::State) -> Result<Review> {
|
||||
review_for(tx, s, &case_witness()).await
|
||||
}
|
||||
async fn review_for(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
s: &crate::State,
|
||||
expected: &Value,
|
||||
) -> Result<Review> {
|
||||
crate::agent_service::control_lock(tx, TENANT, AGENT).await?;
|
||||
let dev=sqlx::query("SELECT asset_id,revoked_at IS NULL AS active,enrollment_generation,assignment_version,connection_boot_id,restore_blocked,sequence_regressed,sync_recovery_required FROM agent_installations WHERE tenant_id=$1 AND agent_id=$2 FOR UPDATE")
|
||||
.bind(TENANT).bind(AGENT).fetch_one(&mut **tx).await?;
|
||||
if dev.get::<Uuid, _>("asset_id") != ASSET
|
||||
|| !dev.get::<bool, _>("active")
|
||||
|| dev.get::<i64, _>("enrollment_generation") != 1
|
||||
|| dev.get::<i64, _>("assignment_version") != 1
|
||||
|| dev.get::<bool, _>("restore_blocked")
|
||||
|| dev.get::<bool, _>("sequence_regressed")
|
||||
|| dev.get::<bool, _>("sync_recovery_required")
|
||||
{
|
||||
return Err(mismatch());
|
||||
}
|
||||
let row = sqlx::query(
|
||||
"SELECT receipt,signature,receipt_id FROM windows_echo_probe_scopes WHERE tenant_id=$1",
|
||||
)
|
||||
.bind(TENANT)
|
||||
.fetch_one(&mut **tx)
|
||||
.await?;
|
||||
let raw: Vec<u8> = row.get("receipt");
|
||||
let sig: Vec<u8> = row.get("signature");
|
||||
verify(s, &raw, &sig)?;
|
||||
let probe: ProbeReceipt = serde_json::from_slice(&raw)?;
|
||||
if !probe.matches_reviewed_witness(expected, probe.issued_at)
|
||||
|| probe.receipt_id != row.get::<Uuid, _>("receipt_id")
|
||||
{
|
||||
return Err(mismatch());
|
||||
}
|
||||
let scope = &probe.scope;
|
||||
let job=sqlx::query("SELECT state,payload,payload_hash,result FROM jobs WHERE tenant_id=$1 AND agent_id=$2 AND asset_id=$3 AND job_id=$4 AND attempt_id=$5 AND operation_id=$6 FOR SHARE")
|
||||
.bind(TENANT).bind(AGENT).bind(ASSET).bind(scope.job_id).bind(scope.attempt_id).bind(scope.operation_id).fetch_one(&mut **tx).await?;
|
||||
let mut outcome: Value = job.get::<Option<Value>, _>("result").ok_or_else(mismatch)?;
|
||||
outcome
|
||||
.as_object_mut()
|
||||
.ok_or_else(mismatch)?
|
||||
.remove("grant_id");
|
||||
outcome["state"] = json!(job.get::<String, _>("state"));
|
||||
let job_bytes: Vec<u8> = job.get("payload");
|
||||
let payload: JobPayload = serde_json::from_slice(&job_bytes)?;
|
||||
if job_bytes != probe.job_payload_bytes
|
||||
|| !scope.matches_job(&payload)
|
||||
|| job.get::<String, _>("payload_hash") != scope.job_payload_hash
|
||||
|| !successful_outcome(&outcome)
|
||||
{
|
||||
return Err(mismatch());
|
||||
}
|
||||
let extra: bool = sqlx::query_scalar(
|
||||
"SELECT EXISTS(SELECT FROM jobs WHERE tenant_id=$1 AND agent_id=$2 AND job_id!=$3)",
|
||||
)
|
||||
.bind(TENANT)
|
||||
.bind(AGENT)
|
||||
.bind(scope.job_id)
|
||||
.fetch_one(&mut **tx)
|
||||
.await?;
|
||||
let existing =
|
||||
sqlx::query("SELECT payload,signature FROM windows_echo_completions WHERE tenant_id=$1")
|
||||
.bind(TENANT)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?;
|
||||
// Only the first transition requires the original single-job inventory.
|
||||
if extra && existing.is_none() {
|
||||
return Err(mismatch());
|
||||
}
|
||||
let grant=sqlx::query("SELECT grant_id,payload,signature FROM execution_permits WHERE tenant_id=$1 AND job_id=$2 AND attempt_id=$3")
|
||||
.bind(TENANT).bind(scope.job_id).bind(scope.attempt_id).fetch_one(&mut **tx).await?;
|
||||
let graw: Vec<u8> = grant.get("payload");
|
||||
let gsig: Vec<u8> = grant.get("signature");
|
||||
verify(s, &graw, &gsig)?;
|
||||
let g: ExecutionPermitPayload = serde_json::from_slice(&graw)?;
|
||||
if g.domain != "rmm.execution-permit"
|
||||
|| g.schema_version != 1
|
||||
|| g.action != "run_script"
|
||||
|| g.agent_update_plan_id.is_some()
|
||||
|| (g.enrollment_generation, g.assignment_version)
|
||||
!= (scope.enrollment_generation, scope.assignment_version)
|
||||
|| g.max_runtime_seconds != scope.max_runtime_seconds
|
||||
|| g.not_before < scope.not_before
|
||||
|| g.start_before > scope.start_before
|
||||
|| g.not_before >= g.start_before
|
||||
|| g.start_before - g.not_before > Duration::seconds(60)
|
||||
|| g.grant_id != grant.get::<Uuid, _>("grant_id")
|
||||
|| g.job_id != scope.job_id
|
||||
|| g.attempt_id != scope.attempt_id
|
||||
|| g.plan_hash != scope.plan_hash
|
||||
|| g.job_payload_hash != scope.job_payload_hash
|
||||
|| g.agent_id != AGENT
|
||||
|| g.tenant_id != TENANT
|
||||
|| g.asset_id != ASSET
|
||||
|| g.execution_identity != "agent_service"
|
||||
|| g.key_id != s.job_signer.key_id
|
||||
|| job.get::<Option<Value>, _>("result").unwrap_or_default()["grant_id"]
|
||||
!= g.grant_id.to_string()
|
||||
{
|
||||
return Err(mismatch());
|
||||
}
|
||||
let authorized:bool=sqlx::query_scalar("SELECT EXISTS(SELECT FROM authorizations WHERE tenant_id=$1 AND authorization_id=$2 AND operation_id=$3 AND plan_id=$4 AND plan_hash=$5 AND principal=$6 AND approval_origin='dev_test_browser_simulation' AND testsimulation AND revoked_at IS NULL)")
|
||||
.bind(TENANT).bind(g.authorization_id).bind(scope.operation_id).bind(scope.plan_id).bind(&scope.plan_hash).bind(TEST_PRINCIPAL).fetch_one(&mut **tx).await?;
|
||||
if !authorized {
|
||||
return Err(mismatch());
|
||||
}
|
||||
let boot: Uuid = serde_json::from_value(expected["boot_id"].clone())?;
|
||||
if dev.get::<Option<Uuid>, _>("connection_boot_id") != Some(boot) {
|
||||
return Err(mismatch());
|
||||
}
|
||||
let witness = Witness {
|
||||
scope: scope.clone(),
|
||||
probe_receipt_id: probe.receipt_id,
|
||||
probe_receipt_sha256: hash(&raw),
|
||||
probe_signature_sha256: hash(&sig),
|
||||
grant_id: g.grant_id,
|
||||
grant_payload_sha256: hash(&graw),
|
||||
grant_signature_sha256: hash(&gsig),
|
||||
outcome_sha256: inventory_sha256(&outcome)?,
|
||||
boot_id: boot,
|
||||
};
|
||||
let existing = existing.map(|r| SignedReceipt {
|
||||
payload: r.get("payload"),
|
||||
signature: r.get("signature"),
|
||||
});
|
||||
if let Some(signed) = &existing {
|
||||
verify(s, &signed.payload, &signed.signature)?;
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
if !r.valid(r.issued_at) || r.witness != witness || r.key_id != s.job_signer.key_id {
|
||||
return Err(mismatch());
|
||||
}
|
||||
}
|
||||
Ok(Review { witness, existing })
|
||||
}
|
||||
async fn read(
|
||||
State(s): State<crate::State>,
|
||||
Path(t): Path<Uuid>,
|
||||
h: HeaderMap,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = access(&s, &h, t, false).await?;
|
||||
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
|
||||
crate::permissions::require_principal(&mut tx, &p, "operations:read").await?;
|
||||
let result = review(&mut tx, &s).await?.view();
|
||||
tx.commit().await?;
|
||||
Ok(Json(result))
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Confirmation {
|
||||
reviewed: bool,
|
||||
witness_sha256: String,
|
||||
}
|
||||
async fn confirm(
|
||||
State(s): State<crate::State>,
|
||||
Path(t): Path<Uuid>,
|
||||
h: HeaderMap,
|
||||
Json(input): Json<Confirmation>,
|
||||
) -> Result<Json<Value>> {
|
||||
let p = access(&s, &h, t, true).await?;
|
||||
confirm_review(&s, &p, input, &case_witness()).await
|
||||
}
|
||||
async fn confirm_review(
|
||||
s: &crate::State,
|
||||
p: &Principal,
|
||||
input: Confirmation,
|
||||
expected: &Value,
|
||||
) -> Result<Json<Value>> {
|
||||
let t = p.tenant;
|
||||
let session = browser(p)?;
|
||||
if !input.reviewed {
|
||||
return Err(Error(StatusCode::BAD_REQUEST, "REVIEW_REQUIRED"));
|
||||
}
|
||||
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
|
||||
let allowed: bool = sqlx::query_scalar("SELECT lock_windows_echo_probe_reviewer()")
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
if !allowed {
|
||||
return Err(Error(StatusCode::FORBIDDEN, "DEV_TEST_WEBUI_REQUIRED"));
|
||||
}
|
||||
let mut r = review_for(&mut tx, s, expected).await?;
|
||||
if r.witness.digest().as_deref() != Some(input.witness_sha256.as_str()) {
|
||||
return Err(mismatch());
|
||||
}
|
||||
if r.existing.is_none() {
|
||||
// SQLx binds PostgreSQL timestamps at microsecond precision. Normalize
|
||||
// before signing so JSON's parser cannot round a nanosecond differently.
|
||||
let now = Utc::now().trunc_subsecs(6);
|
||||
let receipt = Receipt {
|
||||
domain: DOMAIN.into(),
|
||||
schema_version: 1,
|
||||
receipt_id: Uuid::new_v4(),
|
||||
witness: r.witness.clone(),
|
||||
witness_sha256: input.witness_sha256,
|
||||
key_id: s.job_signer.key_id.clone(),
|
||||
issued_at: now,
|
||||
valid_until: now + Duration::minutes(10),
|
||||
confirmed_by: p.name.clone(),
|
||||
approval_origin: p.approval_origin().into(),
|
||||
testsimulation: true,
|
||||
ready: false,
|
||||
helper_ready: false,
|
||||
unlock: false,
|
||||
};
|
||||
if !receipt.valid(now) {
|
||||
return Err(mismatch());
|
||||
}
|
||||
let raw = serde_json::to_vec(&receipt)?;
|
||||
let signature = s.job_signer.sign(&raw)?;
|
||||
sqlx::query("INSERT INTO windows_echo_completions(tenant_id,receipt_id,witness_sha256,payload,signature,confirmed_by,session_hash,issued_at,valid_until) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)")
|
||||
.bind(t).bind(receipt.receipt_id).bind(&receipt.witness_sha256).bind(&raw).bind(&signature).bind(&p.name).bind(session).bind(now).bind(receipt.valid_until).execute(&mut *tx).await?;
|
||||
sqlx::query("INSERT INTO audit_events(tenant_id,event_id,principal,event_type,data) VALUES($1,$2,$3,'windows_echo_scope_completed',$4)")
|
||||
.bind(t).bind(receipt.receipt_id).bind(&p.name).bind(json!({"witness":receipt.witness,"witness_sha256":receipt.witness_sha256,
|
||||
"receipt_sha256":hash(&raw),"approval_origin":receipt.approval_origin,"testsimulation":true,"ready":false,"helper_ready":false,"unlock":false})).execute(&mut *tx).await?;
|
||||
r.existing = Some(SignedReceipt {
|
||||
payload: raw,
|
||||
signature,
|
||||
});
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(Json(r.view()))
|
||||
}
|
||||
async fn download(
|
||||
State(s): State<crate::State>,
|
||||
Path(t): Path<Uuid>,
|
||||
h: HeaderMap,
|
||||
) -> Result<Json<SignedReceipt>> {
|
||||
let p = access(&s, &h, t, false).await?;
|
||||
browser(&p)?;
|
||||
let mut tx = rmm_persistence::tenant_tx(&s.pool, t).await?;
|
||||
crate::permissions::require_principal(&mut tx, &p, "plans:approve").await?;
|
||||
let signed = review(&mut tx, &s)
|
||||
.await?
|
||||
.existing
|
||||
.ok_or(Error(StatusCode::NOT_FOUND, "NOT_FOUND"))?;
|
||||
let r: Receipt = serde_json::from_slice(&signed.payload)?;
|
||||
if !r.valid(Utc::now()) {
|
||||
return Err(Error(
|
||||
StatusCode::CONFLICT,
|
||||
"WINDOWS_ECHO_COMPLETION_EXPIRED",
|
||||
));
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(Json(signed))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn test_completed_echo(
|
||||
admin: &sqlx::PgPool,
|
||||
s: &crate::State,
|
||||
p: &Principal,
|
||||
expected: &Value,
|
||||
) {
|
||||
let nanosecond = chrono::DateTime::from_timestamp(1794820000, 123456789).unwrap();
|
||||
let compare = |t: chrono::DateTime<Utc>| {
|
||||
sqlx::query_scalar::<_, bool>("SELECT $1::timestamptz=$2::text::timestamptz")
|
||||
.bind(t)
|
||||
.bind(t.to_rfc3339())
|
||||
};
|
||||
assert!(!compare(nanosecond).fetch_one(admin).await.unwrap());
|
||||
assert!(
|
||||
compare(nanosecond.trunc_subsecs(6))
|
||||
.fetch_one(admin)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
let row =
|
||||
sqlx::query("SELECT job_id,attempt_id,grant_id FROM execution_permits WHERE tenant_id=$1")
|
||||
.bind(TENANT)
|
||||
.fetch_one(admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let grant: Uuid = row.get("grant_id");
|
||||
let job: Uuid = row.get("job_id");
|
||||
let original_counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM jobs),(SELECT count(*) FROM authorizations),(SELECT count(*) FROM execution_permits)").fetch_one(admin).await.unwrap();
|
||||
let confirm = |digest: String| Confirmation {
|
||||
reviewed: true,
|
||||
witness_sha256: digest,
|
||||
};
|
||||
assert!(
|
||||
confirm_review(s, p, confirm("0".repeat(64)), expected)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut no_browser = p.clone();
|
||||
no_browser.session_hash = None;
|
||||
assert_eq!(
|
||||
confirm_review(s, &no_browser, confirm("0".repeat(64)), expected)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.0,
|
||||
StatusCode::FORBIDDEN
|
||||
);
|
||||
let outcome = json!({"exit_code":0,"stdout":rmm_domain::windows_echo_probe::OUTPUT.as_bytes(),"stderr":[],"error_code":"","output_truncated":false,"reboot_required":false,
|
||||
"evidence":{"execution_sid":"S-1-5-19","execution_policy_scope":"process","execution_policy_requested":"RemoteSigned","postconditions_passed":true,
|
||||
"script_sha256":hash(rmm_domain::windows_echo_probe::SCRIPT.as_bytes())},"grant_id":grant});
|
||||
sqlx::query("UPDATE jobs SET state='succeeded',result=$2 WHERE job_id=$1")
|
||||
.bind(job)
|
||||
.bind(outcome)
|
||||
.execute(admin)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("UPDATE agent_installations SET restore_blocked=false WHERE agent_id=$1")
|
||||
.bind(AGENT)
|
||||
.execute(admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut tx = rmm_persistence::tenant_tx(&s.pool, TENANT).await.unwrap();
|
||||
let r = review_for(&mut tx, s, expected).await.unwrap();
|
||||
tx.rollback().await.unwrap();
|
||||
let digest = r.witness.digest().unwrap();
|
||||
assert!(
|
||||
confirm_review(s, p, confirm("0".repeat(64)), expected)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let first = confirm_review(s, p, confirm(digest.clone()), expected)
|
||||
.await
|
||||
.unwrap()
|
||||
.0;
|
||||
assert_eq!(first["receipt"]["ready"], false);
|
||||
assert_eq!(first["receipt"]["helper_ready"], false);
|
||||
assert_eq!(first["receipt"]["unlock"], false);
|
||||
assert_eq!(first["receipt"]["testsimulation"], true);
|
||||
assert_eq!(
|
||||
confirm_review(s, p, confirm(digest), expected)
|
||||
.await
|
||||
.unwrap()
|
||||
.0,
|
||||
first
|
||||
);
|
||||
let counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM jobs),(SELECT count(*) FROM authorizations),(SELECT count(*) FROM execution_permits)").fetch_one(admin).await.unwrap();
|
||||
assert_eq!(counts, original_counts);
|
||||
assert!(
|
||||
sqlx::query("DELETE FROM windows_echo_completions")
|
||||
.execute(admin)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
sqlx::query("UPDATE windows_echo_completions SET witness_sha256=repeat('0',64)")
|
||||
.execute(admin)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM windows_echo_probe_scopes")
|
||||
.fetch_one(admin)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>(
|
||||
"SELECT count(*) FROM audit_events WHERE event_type='windows_echo_scope_completed'"
|
||||
)
|
||||
.fetch_one(admin)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user