Recover a Windows maintenance setup interrupted before policy activation

This commit is contained in:
2026-11-17 17:32:17 +01:00
parent fe55995154
commit 8c39e79a79
9 changed files with 303 additions and 63 deletions
+92 -28
View File
@@ -65,31 +65,32 @@ def public_key():
conn.close()
def observation(ids, opened, cutoff):
def observation(ids, opened, cutoff=None):
"""Only canonical UUIDs and normalized datetimes reach the query."""
tenant, asset, agent = (ids[k] for k in ('tenant_id', 'asset_id', 'agent_id'))
horizon = f"BETWEEN '{opened}' AND '{cutoff}'" if cutoff else f">= '{opened}'"
return f"""
SELECT jsonb_build_object(
'open_jobs',(SELECT count(*) FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id
AND j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,'head_created_at',c.created_at,
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint,
'signed',encode(c.payload,'hex'),'signature',encode(c.signature,'hex'),
'jobs',COALESCE((SELECT jsonb_agg(jsonb_build_object(
'job',j.job_id,'attempt',j.attempt_id,'grant',g.grant_id,'state',j.state,
'payload_sha256',j.payload_hash,'bytes_hash',encode(sha256(j.payload),'hex'),
'payload_binding',p-'action','action',p->'action',
'principal',z.principal,'window_approval',z.created_at BETWEEN '{opened}' AND '{cutoff}',
'principal',z.principal,'window_approval',z.created_at {horizon},
'dev',z.testsimulation AND z.approval_origin='dev_test_browser_simulation',
'plan_hash',z.plan_hash,'authorization_id',z.authorization_id,
'in_window',g.created_at BETWEEN '{opened}' AND '{cutoff}',
'in_window',g.created_at {horizon},
'reserved',j.start_reserved,'result_grant',j.result->>'grant_id','revision',j.revision)
ORDER BY j.job_id)
FROM jobs j JOIN authorizations z USING(tenant_id,operation_id)
CROSS JOIN LATERAL (SELECT convert_from(j.payload,'UTF8')::jsonb AS p) jp
LEFT JOIN execution_permits g USING(tenant_id,job_id,attempt_id)
WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND p->>'execution_identity'='system'
AND (z.created_at BETWEEN '{opened}' AND '{cutoff}' OR g.created_at BETWEEN '{opened}' AND '{cutoff}')),
AND (z.created_at {horizon} OR g.created_at {horizon})),
'[]'::jsonb)) AS observation
FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
JOIN sync_checkpoints c USING(tenant_id,agent_id,decision_id)
@@ -145,6 +146,63 @@ def verify(snapshot, ids):
return {**ids, **{k:snapshot[k] for k in (*fields,'helper_id')}, 'attempts':attempts}
def prepared_baseline(case, data, ids):
path = case/'prepared.json'
prepared = json.loads(private_read(path))
require(prepared['schema_version'] == 1 and all(prepared[k] == value for k,value in ids.items())
and prepared['release_id'] == scope.identifier(data['release_id']), 'Prepared device or release differs')
for k in ('agent_sha256','helper_binary_sha256'):
require(re.fullmatch('[a-f0-9]{64}', prepared[k]) and data.get(k,prepared[k]) == prepared[k],
'Prepared component differs')
journal = scope.identifier(prepared['journal_id'])
decision = scope.identifier(prepared['decision_id'])
checkpoint = prepared['checkpoint']
require(type(checkpoint) is int and checkpoint >= 0, 'Invalid prepared checkpoint')
opened = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
signed = sql(f"""SELECT json_build_object('signed',encode(payload,'hex'),'signature',encode(signature,'hex'))
FROM sync_checkpoints WHERE tenant_id='{ids['tenant_id']}' AND agent_id='{ids['agent_id']}'
AND journal_id='{journal}' AND checkpoint={checkpoint} AND decision_id='{decision}'
AND created_at<='{opened.isoformat()}'""")
raw = bytes.fromhex(signed['signed'])
key_id, public = public_key()
public.verify(bytes.fromhex(signed['signature']), raw)
baseline = json.loads(raw)
require(baseline['domain']=='rmm.reconciliation-decision' and baseline['schema_version']==1
and baseline['key_id']==key_id and baseline['ready'] is True and baseline['reason']=='CONSISTENT'
and baseline['helper_journal_id'] is None and baseline['helper_checkpoint']==0
and baseline['enrollment_generation']==baseline['assignment_version']==1
and all(baseline[k]==prepared[k] for k in ('tenant_id','agent_id','asset_id','journal_id','checkpoint','decision_id')),
'Prepared baseline must be the signed decision of the normal device without a helper')
return prepared, opened
def require_native_verification(data, witness):
proof = data.get('native_verification') or {}
require(data.get('witness') == witness and proof.get('verified') is True and proof.get('applied') is False
and all(proof.get(k)==witness[k] for k in ('case_id','helper_id','decision_id'))
and type(proof.get('attempts')) is int and proof['attempts']==len(witness['attempts']),
'Exact witness and positive native archive verification required before detach')
def window_query(window, case, ids, data):
policy = json.loads(private_read(window))
only, = policy['scopes']
hashes = ('helper_binary_sha256','helper_policy_sha256')
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
windows_update_max_runtime_seconds=3600)
for k in hashes:
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
expected_scope[k] = only[k]
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
'Case identity, privileges or original policy differ')
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
require(opened <= cutoff, 'Invalid maintenance horizon')
return observation(ids,opened.isoformat(),cutoff.isoformat())
def server(action, data):
require(action in ('witness','detach'), 'Only witness and detach are supported')
ids = {k:scope.identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id')}
@@ -163,35 +221,29 @@ def server(action, data):
mode = case.lstat()
require(case.resolve() == case and stat.S_ISDIR(mode.st_mode) and mode.st_uid == 0
and stat.S_IMODE(mode.st_mode) == 0o700, 'Unsafe development window directory')
if not window.exists():
# Even a failed opening can already have registered a real helper journal.
if not window.exists() and not (case/'prepared.json').exists():
# The operator persists its local case ID before prepare. A crash in
# that gap has no authority to detach any helper or alter old jobs.
require(action == 'witness' and data.get('native_window_absent') is True
and private_read(case/'closed') == b'closed\n', 'Unprepared case needs a closed, empty native window')
original = case/'original.json'
require(not original.exists() or private_read(original)==current, 'Policy changed during incomplete preparation')
row = sql(f"""SELECT json_build_object('empty',EXISTS (
SELECT 1 FROM agent_installations i LEFT JOIN agent_sync_state s USING(tenant_id,agent_id)
SELECT 1 FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
WHERE i.tenant_id='{ids['tenant_id']}' AND i.agent_id='{ids['agent_id']}' AND i.asset_id='{ids['asset_id']}'
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND coalesce(s.helper_checkpoint,0)=0
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND s.helper_checkpoint=0
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND
(j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')
OR convert_from(j.payload,'UTF8')::jsonb->>'execution_identity'='system'))))""")
require(row['empty'] and action == 'witness', 'A helper or SYSTEM history exists without an opened scope')
j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))))""")
require(row['empty'], 'Unprepared case still has a helper or an unfinished job')
return None
policy = json.loads(private_read(window))
require(private_read(case/'closed') == b'closed\n' and private_read(case/'original.json') == current,
'Expected this closed scope and its original policy')
only, = policy['scopes']
hashes = ('helper_binary_sha256','helper_policy_sha256')
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
windows_update_max_runtime_seconds=3600)
for k in hashes:
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
expected_scope[k] = only[k]
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
'Case identity, privileges or original policy differ')
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
require(opened <= cutoff, 'Invalid maintenance horizon')
query = observation(ids,opened.isoformat(),cutoff.isoformat())
prepared = None
if window.exists():
query = window_query(window, case, ids, data)
else:
prepared, opened = prepared_baseline(case, data, ids)
query = observation(ids, opened.isoformat())
record = sql(query)
require(record['open_jobs'] == 0, 'Open jobs require separate reconciliation')
saved = case/'detach-intent.json'
@@ -200,16 +252,28 @@ def server(action, data):
if done.exists():
require(receipt and json.loads(private_read(done)) == receipt, 'Completion receipt differs from detach intent')
return receipt
if prepared:
require(record['journal_id']==prepared['journal_id'] and record['checkpoint']>=prepared['checkpoint']
and not record['jobs'], 'Prepared journal changed or a SYSTEM job or grant was created after preparation')
if record['helper_id'] is not None:
require(record['decision_id'] != prepared['decision_id'] and scope.stamp(record['head_created_at'])>=opened,
'The current helper was not bound after preparation')
if record['helper_id'] is None:
if prepared and receipt is None and action=='witness' and record['helper_checkpoint']==0:
return None
require(receipt and receipt['journal_id'] == record['journal_id'] and record['helper_checkpoint'] == 0,
'Missing evidence of the original detach transaction')
if action == 'detach':
if prepared:
require_native_verification(data, receipt)
scope.atomic(done,scope.encode(receipt))
return receipt
witness = verify(record,ids)
require(receipt in (None,witness) and data.get('witness',witness) == witness, 'Device or witness changed')
if action == 'witness':
return witness
if prepared:
require_native_verification(data, witness)
if receipt is None:
scope.atomic(saved,scope.encode(witness))
# Fence a concurrent signed sync (same lock order as the gateway). The server-owned