Persist the exact Windows echo scope across recovery and process restarts
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -166,11 +166,19 @@ mod windows {
|
||||
args.len() == 1
|
||||
&& matches!(
|
||||
args[0].as_str(),
|
||||
"prepare" | "read" | "verify-receipt" | "apply"
|
||||
"prepare"
|
||||
| "read"
|
||||
| "verify-receipt"
|
||||
| "apply"
|
||||
| "echo-read"
|
||||
| "echo-verify-receipt"
|
||||
| "echo-apply"
|
||||
),
|
||||
"use prepare, read, verify-receipt or apply"
|
||||
);
|
||||
let apply = args[0] == "apply";
|
||||
let echo = args[0].starts_with("echo-");
|
||||
let mode = args[0].strip_prefix("echo-").unwrap_or(&args[0]);
|
||||
let apply = mode == "apply";
|
||||
let key_readers = [ipc::service_sid(SERVICE_NAME)?];
|
||||
let _root = PinnedFile::open(Path::new(ROOT), true, &key_readers, &[], false)?;
|
||||
let mut binding_file = PinnedFile::open(
|
||||
@@ -196,16 +204,28 @@ mod windows {
|
||||
&[],
|
||||
false,
|
||||
)?;
|
||||
ensure!(
|
||||
config.sha256()? == PASSIVE,
|
||||
"operator must restore passive config first"
|
||||
);
|
||||
if echo {
|
||||
let expected = serde_json::json!({
|
||||
"server_name":"discovery.ease-technologies.de","connect_address":"88.99.80.202:443",
|
||||
"ca_file":Path::new(ROOT).join("server-ca.pem"),"state_directory":STATE,
|
||||
"enrollment_token_file":Path::new(STATE).join("enrollment.token"),"transport":"auto",
|
||||
"job_binding_file":Path::new(ROOT).join("binding.json"),"enable_jobs":true,
|
||||
"windows_execution_profile":"powershell_51_remote_signed","windows_echo_probe_only":true});
|
||||
let actual: serde_json::Value =
|
||||
serde_json::from_slice(&data(&mut config.file, 32768)?)?;
|
||||
ensure!(actual == expected, "expected exact Echo-only configuration");
|
||||
} else {
|
||||
ensure!(
|
||||
config.sha256()? == PASSIVE,
|
||||
"operator must restore passive config first"
|
||||
);
|
||||
}
|
||||
let boot = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()?
|
||||
.block_on(native::boot_id())?;
|
||||
ensure!(boot == BOOT, "unexpected Windows boot");
|
||||
let review = if matches!(args[0].as_str(), "verify-receipt" | "apply") {
|
||||
let review = if matches!(mode, "verify-receipt" | "apply") {
|
||||
let _folder = PinnedFile::open(
|
||||
Path::new(RECEIPT).parent().context("bootstrap path")?,
|
||||
true,
|
||||
@@ -214,12 +234,13 @@ mod windows {
|
||||
false,
|
||||
)?;
|
||||
let mut file = PinnedFile::open(Path::new(RECEIPT), false, &[], &[], false)?;
|
||||
Some(Reviewed::verify(
|
||||
data(&mut file.file, 65536)?,
|
||||
&agent,
|
||||
&settings.binding,
|
||||
&witness,
|
||||
)?)
|
||||
let bytes = data(&mut file.file, 65536)?;
|
||||
Some(if echo {
|
||||
let scope = rmm_agent_windows::echo_probe::protected_scope()?;
|
||||
Reviewed::verify_echo(bytes, &agent, &settings.binding, &witness, &scope)?
|
||||
} else {
|
||||
Reviewed::verify(bytes, &agent, &settings.binding, &witness)?
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -315,7 +336,10 @@ mod windows {
|
||||
);
|
||||
if let Some(review) = &review {
|
||||
review.timely(chrono::Utc::now())?;
|
||||
println!("valid human review for the verified local history");
|
||||
println!(
|
||||
"valid {} review for the verified local history",
|
||||
if echo { "simulated Echo-only" } else { "human" }
|
||||
);
|
||||
}
|
||||
println!(
|
||||
"verified contracted history; fault={}",
|
||||
|
||||
@@ -13,22 +13,34 @@ use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params, types
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use rmm_agent_windows::echo_probe;
|
||||
use rmm_domain::windows_echo_probe::{ProbeReceipt, ProbeScope};
|
||||
use uuid::Uuid;
|
||||
const ECHO_AUDIT: &str = "windows_echo_checkpoint_recovery";
|
||||
const AUDIT: &str = "windows_human_checkpoint_recovery";
|
||||
const AUDIT_SQL: &str = "CREATE TABLE windows_human_checkpoint_recovery(id INTEGER PRIMARY KEY CHECK(id=1),review_id TEXT NOT NULL,receipt_id TEXT NOT NULL,old_fault TEXT NOT NULL,original_receipt BLOB NOT NULL,inventory_sha256 TEXT NOT NULL,applied_at TEXT NOT NULL)";
|
||||
struct ReviewFields {
|
||||
request_id: Uuid,
|
||||
receipt_id: Uuid,
|
||||
issued_at: DateTime<Utc>,
|
||||
valid_until: DateTime<Utc>,
|
||||
witness: Value,
|
||||
}
|
||||
pub struct Reviewed {
|
||||
receipt: Receipt,
|
||||
receipt: ReviewFields,
|
||||
original: Vec<u8>,
|
||||
echo: Option<(ProbeScope, Vec<u8>)>,
|
||||
}
|
||||
|
||||
impl Reviewed {
|
||||
pub fn verify(
|
||||
raw: Vec<u8>,
|
||||
fn signed(
|
||||
raw: &[u8],
|
||||
agent: &Binding,
|
||||
helper: &Binding,
|
||||
expected: &Value,
|
||||
) -> Result<Self> {
|
||||
) -> Result<SignedReceipt> {
|
||||
ensure!(raw.len() <= 65536, "recovery envelope size");
|
||||
let envelope: SignedReceipt = serde_json::from_slice(&raw)?;
|
||||
let envelope: SignedReceipt = serde_json::from_slice(raw)?;
|
||||
ensure!(
|
||||
envelope.payload.len() <= 16384 && envelope.signature.len() == 64,
|
||||
"recovery signature size"
|
||||
@@ -49,6 +61,15 @@ impl Reviewed {
|
||||
"recovery signature"
|
||||
);
|
||||
}
|
||||
Ok(envelope)
|
||||
}
|
||||
pub fn verify(
|
||||
raw: Vec<u8>,
|
||||
agent: &Binding,
|
||||
helper: &Binding,
|
||||
expected: &Value,
|
||||
) -> Result<Self> {
|
||||
let envelope = Self::signed(&raw, agent, helper, expected)?;
|
||||
let receipt: Receipt = serde_json::from_slice(&envelope.payload)?;
|
||||
// Historical validity is sufficient for looking up a previously committed receipt.
|
||||
ensure!(
|
||||
@@ -56,8 +77,40 @@ impl Reviewed {
|
||||
"historical human review"
|
||||
);
|
||||
Ok(Self {
|
||||
receipt,
|
||||
receipt: ReviewFields {
|
||||
request_id: receipt.request_id,
|
||||
receipt_id: receipt.receipt_id,
|
||||
issued_at: receipt.issued_at,
|
||||
valid_until: receipt.valid_until,
|
||||
witness: receipt.witness,
|
||||
},
|
||||
original: raw,
|
||||
echo: None,
|
||||
})
|
||||
}
|
||||
pub fn verify_echo(
|
||||
raw: Vec<u8>,
|
||||
agent: &Binding,
|
||||
helper: &Binding,
|
||||
expected: &Value,
|
||||
configured: &ProbeScope,
|
||||
) -> Result<Self> {
|
||||
let envelope = Self::signed(&raw, agent, helper, expected)?;
|
||||
let r: ProbeReceipt = serde_json::from_slice(&envelope.payload)?;
|
||||
ensure!(
|
||||
r.matches_reviewed_witness(expected, r.issued_at) && &r.scope == configured,
|
||||
"simulated echo scope/receipt"
|
||||
);
|
||||
Ok(Self {
|
||||
receipt: ReviewFields {
|
||||
request_id: r.request_id,
|
||||
receipt_id: r.receipt_id,
|
||||
issued_at: r.issued_at,
|
||||
valid_until: r.valid_until,
|
||||
witness: r.witness,
|
||||
},
|
||||
original: raw,
|
||||
echo: Some((r.scope, r.job_payload_bytes)),
|
||||
})
|
||||
}
|
||||
pub fn timely(&self, now: DateTime<Utc>) -> Result<()> {
|
||||
@@ -84,7 +137,11 @@ fn include(hash: &mut Sha256, tag: u8, bytes: &[u8]) {
|
||||
}
|
||||
|
||||
/// Compare the full logical contents and schema, excluding only the receipt and normalized fault.
|
||||
#[cfg(test)]
|
||||
fn inventory(db: &Connection) -> Result<String> {
|
||||
inventory_for(db, false)
|
||||
}
|
||||
fn inventory_for(db: &Connection, echo: bool) -> Result<String> {
|
||||
let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?;
|
||||
ensure!(check == "ok", "journal integrity");
|
||||
let mut statement = db.prepare(
|
||||
@@ -97,6 +154,9 @@ fn inventory(db: &Connection) -> Result<String> {
|
||||
let kind: String = row.get(0)?;
|
||||
let name: String = row.get(1)?;
|
||||
let table: String = row.get(2)?;
|
||||
if echo && (echo_probe::OBJECTS.contains(&name.as_str()) || name == ECHO_AUDIT) {
|
||||
continue;
|
||||
}
|
||||
let sql: Option<String> = row.get(3)?;
|
||||
ensure!(
|
||||
matches!(kind.as_str(), "table" | "index" | "trigger")
|
||||
@@ -174,25 +234,37 @@ impl LocalJournals<'_> {
|
||||
let tx = self
|
||||
.agent
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let echo = reviewed.echo.is_some();
|
||||
let audit = if echo { ECHO_AUDIT } else { AUDIT };
|
||||
let ddl = AUDIT_SQL.replace(AUDIT, audit);
|
||||
let other = if echo { AUDIT } else { ECHO_AUDIT };
|
||||
ensure!(
|
||||
!tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=?)",
|
||||
[other],
|
||||
|r| r.get::<_, bool>(0)
|
||||
)?,
|
||||
"another recovery mode already exists"
|
||||
);
|
||||
let prior: Option<String> = tx
|
||||
.query_row(
|
||||
"SELECT sql FROM sqlite_schema WHERE name=?1",
|
||||
[AUDIT],
|
||||
[audit],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
let applied = prior.is_some();
|
||||
if let Some(schema) = prior {
|
||||
ensure!(
|
||||
schema == AUDIT_SQL
|
||||
schema == ddl
|
||||
&& tx.query_row(
|
||||
"SELECT count(*) FROM sqlite_schema WHERE tbl_name=?1",
|
||||
[AUDIT],
|
||||
[audit],
|
||||
|r| r.get::<_, usize>(0)
|
||||
)? == 1,
|
||||
"foreign archive"
|
||||
);
|
||||
let identical:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM windows_human_checkpoint_recovery WHERE id=1 AND review_id=? AND receipt_id=? AND old_fault=? AND original_receipt=?)",params![reviewed.receipt.request_id.to_string(),reviewed.receipt.receipt_id.to_string(),FAULT,reviewed.original],|r|r.get(0))?;
|
||||
let identical:bool=tx.query_row(&format!("SELECT EXISTS(SELECT 1 FROM {audit} WHERE id=1 AND review_id=? AND receipt_id=? AND old_fault=? AND original_receipt=?)"),params![reviewed.receipt.request_id.to_string(),reviewed.receipt.receipt_id.to_string(),FAULT,reviewed.original],|r|r.get(0))?;
|
||||
ensure!(identical, "another recovery already exists");
|
||||
}
|
||||
let original = verify_local_history(
|
||||
@@ -241,18 +313,27 @@ impl LocalJournals<'_> {
|
||||
}
|
||||
if applied {
|
||||
let saved: String = tx.query_row(
|
||||
"SELECT inventory_sha256 FROM windows_human_checkpoint_recovery WHERE id=1",
|
||||
&format!("SELECT inventory_sha256 FROM {audit} WHERE id=1"),
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(inventory(&tx)? == saved, "archive inventory changed");
|
||||
if let Some((scope, bytes)) = &reviewed.echo {
|
||||
echo_probe::check(&tx, scope, bytes, &reviewed.original)?;
|
||||
}
|
||||
ensure!(
|
||||
inventory_for(&tx, echo)? == saved,
|
||||
"archive inventory changed"
|
||||
);
|
||||
return Ok("already applied");
|
||||
}
|
||||
reviewed.timely(now())?;
|
||||
let before = inventory(&tx)?;
|
||||
tx.execute(AUDIT_SQL, [])?;
|
||||
let before = inventory_for(&tx, echo)?;
|
||||
if let Some((scope, bytes)) = &reviewed.echo {
|
||||
echo_probe::install(&tx, scope, bytes, &reviewed.original)?;
|
||||
}
|
||||
tx.execute(&ddl, [])?;
|
||||
tx.execute(
|
||||
"INSERT INTO windows_human_checkpoint_recovery VALUES(1,?,?,?,?,?,?)",
|
||||
&format!("INSERT INTO {audit} VALUES(1,?,?,?,?,?,?)"),
|
||||
params![
|
||||
reviewed.receipt.request_id.to_string(),
|
||||
reviewed.receipt.receipt_id.to_string(),
|
||||
@@ -269,7 +350,7 @@ impl LocalJournals<'_> {
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
cleared && inventory(&tx)? == before,
|
||||
cleared && inventory_for(&tx, echo)? == before,
|
||||
"unrelated journal data changed"
|
||||
);
|
||||
reviewed.timely(now())?;
|
||||
@@ -294,7 +375,7 @@ mod tests {
|
||||
"/../agent-core/tests/support/windows_review_fixture.rs"
|
||||
));
|
||||
}
|
||||
fn signed(r: &Receipt, k: &SigningKey) -> Result<Vec<u8>> {
|
||||
fn signed(r: &impl serde::Serialize, k: &SigningKey) -> Result<Vec<u8>> {
|
||||
let payload = serde_json::to_vec(r)?;
|
||||
Ok(serde_json::to_vec(&SignedReceipt {
|
||||
signature: k.sign(&payload).to_bytes().to_vec(),
|
||||
@@ -302,6 +383,154 @@ mod tests {
|
||||
})?)
|
||||
}
|
||||
#[test]
|
||||
fn scoped_echo_preserves_journals_and_rejects_other_offers() -> Result<()> {
|
||||
use rmm_domain::{
|
||||
jobs::{Action, JobPayload, PlanPayload, PlannedJob, WindowsExecutionProfile},
|
||||
windows_echo_probe as echo,
|
||||
};
|
||||
use rmm_protocol::JobOffer;
|
||||
let (mut a, l, h, binding, key) = data::verified_pair_for(echo::TENANT, echo::ASSET)?;
|
||||
a.execute_batch("DROP TABLE job_attempts; CREATE TABLE job_attempts(job TEXT NOT NULL,attempt TEXT NOT NULL,offer BLOB NOT NULL,state TEXT NOT NULL,PRIMARY KEY(job,attempt))")?;
|
||||
let now = Utc::now();
|
||||
let job = JobPayload {
|
||||
schema_version: 1,
|
||||
tenant_id: binding.tenant_id,
|
||||
asset_id: binding.asset_id,
|
||||
agent_id: binding.agent_id,
|
||||
enrollment_generation: 1,
|
||||
assignment_version: 1,
|
||||
job_id: Uuid::new_v4(),
|
||||
attempt_id: Uuid::new_v4(),
|
||||
action: Action::RunScript {
|
||||
interpreter: "powershell".into(),
|
||||
script_utf8: echo::SCRIPT.into(),
|
||||
arguments: vec![],
|
||||
expected_stdout: Some(echo::OUTPUT.into()),
|
||||
windows_execution_profile: Some(WindowsExecutionProfile::PowerShell51RemoteSigned),
|
||||
privileged_script_profile: None,
|
||||
},
|
||||
execution_identity: "agent_service".into(),
|
||||
not_before: now,
|
||||
start_before: now + Duration::minutes(15),
|
||||
max_runtime_seconds: 60,
|
||||
max_output_bytes: 65536,
|
||||
};
|
||||
let bytes = serde_json::to_vec(&job)?;
|
||||
let plan = PlanPayload {
|
||||
schema_version: 1,
|
||||
plan_id: Uuid::new_v4(),
|
||||
tenant_id: echo::TENANT,
|
||||
principal: format!("automation:{}", echo::TENANT),
|
||||
reason: "fixture".into(),
|
||||
jobs: vec![PlannedJob {
|
||||
payload: job.clone(),
|
||||
job_payload_hash: hash(&bytes),
|
||||
script_sha256: Some(hash(echo::SCRIPT.as_bytes())),
|
||||
}],
|
||||
max_parallel: 1,
|
||||
halt_after_failures: 1,
|
||||
recipe_references: vec![],
|
||||
artifact_references: vec![],
|
||||
};
|
||||
let scope = ProbeScope::from_plan(&plan, Uuid::new_v4()).context("fixed Echo plan")?;
|
||||
let mut w = case_witness();
|
||||
let (j, p): (String, String) = a.query_row(
|
||||
"SELECT journal_id,helper_id FROM reconcile_meta WHERE id=1",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
w["verification_key_id"] = hash(&key.verifying_key().to_bytes()).into();
|
||||
w["agent_journal_id"] = j.into();
|
||||
w["helper_journal_id"] = p.into();
|
||||
w["decisions"] = verify_local_history(&a, &l, &h, &binding, &binding, Some(FAULT))?.into();
|
||||
let receipt = ProbeReceipt {
|
||||
domain: echo::DOMAIN.into(),
|
||||
schema_version: 1,
|
||||
request_id: Uuid::new_v4(),
|
||||
receipt_id: Uuid::new_v4(),
|
||||
witness: w.clone(),
|
||||
witness_sha256: inventory_sha256(&w)?,
|
||||
scope: scope.clone(),
|
||||
scope_sha256: scope.digest().context("scope digest")?,
|
||||
job_payload_bytes: bytes.clone(),
|
||||
key_id: hash(&key.verifying_key().to_bytes()),
|
||||
issued_at: now,
|
||||
valid_until: now + Duration::minutes(10),
|
||||
confirmed_by: echo::TEST_PRINCIPAL.into(),
|
||||
approval_origin: "dev_test_browser_simulation".into(),
|
||||
testsimulation: true,
|
||||
};
|
||||
let raw = signed(&receipt, &key)?;
|
||||
assert!(Reviewed::verify(raw.clone(), &binding, &binding, &w).is_err());
|
||||
let mut different = scope.clone();
|
||||
different.attempt_id = Uuid::new_v4();
|
||||
assert!(Reviewed::verify_echo(raw.clone(), &binding, &binding, &w, &different).is_err());
|
||||
let reviewed = Reviewed::verify_echo(raw, &binding, &binding, &w, &scope)?;
|
||||
let original = inventory(&a)?;
|
||||
let local = inventory(&l)?;
|
||||
let helper = inventory(&h)?;
|
||||
let mut db = LocalJournals {
|
||||
agent: &mut a,
|
||||
consumption: &l,
|
||||
helper: &h,
|
||||
agent_binding: &binding,
|
||||
helper_binding: &binding,
|
||||
};
|
||||
assert!(db.apply(&reviewed, || now + Duration::minutes(11)).is_err());
|
||||
db.apply(&reviewed, || now)?;
|
||||
assert_eq!(inventory_for(db.agent, true)?, original);
|
||||
assert_eq!((inventory(&l)?, inventory(&h)?), (local, helper));
|
||||
assert_eq!(
|
||||
db.apply(&reviewed, || now + Duration::days(1))?,
|
||||
"already applied"
|
||||
);
|
||||
let offer = JobOffer {
|
||||
job_id: scope.job_id.to_string(),
|
||||
attempt_id: scope.attempt_id.to_string(),
|
||||
authorization_id: Uuid::new_v4().to_string(),
|
||||
plan_hash: scope.plan_hash.clone(),
|
||||
job_payload_hash: scope.job_payload_hash.clone(),
|
||||
payload_bytes: bytes,
|
||||
};
|
||||
assert!(echo_probe::permits_offer(&scope, &offer, &job));
|
||||
let store = |o: &JobOffer| {
|
||||
db.agent.execute(
|
||||
"INSERT INTO job_attempts VALUES(?,?,?,'accepted')",
|
||||
params![o.job_id, o.attempt_id, o.encode_to_vec()],
|
||||
)
|
||||
};
|
||||
for field in 0..4 {
|
||||
let mut bad = offer.clone();
|
||||
match field {
|
||||
0 => bad.attempt_id = Uuid::new_v4().to_string(),
|
||||
1 => bad.plan_hash = "0".repeat(64),
|
||||
2 => bad.payload_bytes.push(b' '),
|
||||
_ => bad.job_id = Uuid::new_v4().to_string(),
|
||||
}
|
||||
assert!(!echo_probe::permits_offer(&scope, &bad, &job));
|
||||
assert!(store(&bad).is_err());
|
||||
}
|
||||
assert_eq!(store(&offer)?, 1);
|
||||
assert!(
|
||||
db.agent
|
||||
.execute("UPDATE job_attempts SET offer=x'01'", [])
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
db.agent
|
||||
.execute("UPDATE windows_echo_probe_scope SET job='other'", [])
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
db.agent
|
||||
.execute("DELETE FROM windows_echo_probe_scope", [])
|
||||
.is_err()
|
||||
);
|
||||
assert!(db.agent.execute("INSERT OR REPLACE INTO windows_echo_probe_scope SELECT * FROM windows_echo_probe_scope",[]).is_err());
|
||||
assert!(db.apply(&reviewed, || now).is_err());
|
||||
Ok(())
|
||||
}
|
||||
#[test]
|
||||
fn archived_human_review_preserves_both_journals() -> Result<()> {
|
||||
let (mut a, l, h, binding, key) = data::verified_pair()?;
|
||||
let now = Utc::now();
|
||||
|
||||
Reference in New Issue
Block a user