Recover a Windows maintenance setup interrupted before policy activation

This commit is contained in:
2026-11-17 17:32:17 +01:00
parent fe55995154
commit 8c39e79a79
9 changed files with 303 additions and 63 deletions
+92 -28
View File
@@ -65,31 +65,32 @@ def public_key():
conn.close()
def observation(ids, opened, cutoff):
def observation(ids, opened, cutoff=None):
"""Only canonical UUIDs and normalized datetimes reach the query."""
tenant, asset, agent = (ids[k] for k in ('tenant_id', 'asset_id', 'agent_id'))
horizon = f"BETWEEN '{opened}' AND '{cutoff}'" if cutoff else f">= '{opened}'"
return f"""
SELECT jsonb_build_object(
'open_jobs',(SELECT count(*) FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id
AND j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,'head_created_at',c.created_at,
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint,
'signed',encode(c.payload,'hex'),'signature',encode(c.signature,'hex'),
'jobs',COALESCE((SELECT jsonb_agg(jsonb_build_object(
'job',j.job_id,'attempt',j.attempt_id,'grant',g.grant_id,'state',j.state,
'payload_sha256',j.payload_hash,'bytes_hash',encode(sha256(j.payload),'hex'),
'payload_binding',p-'action','action',p->'action',
'principal',z.principal,'window_approval',z.created_at BETWEEN '{opened}' AND '{cutoff}',
'principal',z.principal,'window_approval',z.created_at {horizon},
'dev',z.testsimulation AND z.approval_origin='dev_test_browser_simulation',
'plan_hash',z.plan_hash,'authorization_id',z.authorization_id,
'in_window',g.created_at BETWEEN '{opened}' AND '{cutoff}',
'in_window',g.created_at {horizon},
'reserved',j.start_reserved,'result_grant',j.result->>'grant_id','revision',j.revision)
ORDER BY j.job_id)
FROM jobs j JOIN authorizations z USING(tenant_id,operation_id)
CROSS JOIN LATERAL (SELECT convert_from(j.payload,'UTF8')::jsonb AS p) jp
LEFT JOIN execution_permits g USING(tenant_id,job_id,attempt_id)
WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND p->>'execution_identity'='system'
AND (z.created_at BETWEEN '{opened}' AND '{cutoff}' OR g.created_at BETWEEN '{opened}' AND '{cutoff}')),
AND (z.created_at {horizon} OR g.created_at {horizon})),
'[]'::jsonb)) AS observation
FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
JOIN sync_checkpoints c USING(tenant_id,agent_id,decision_id)
@@ -145,6 +146,63 @@ def verify(snapshot, ids):
return {**ids, **{k:snapshot[k] for k in (*fields,'helper_id')}, 'attempts':attempts}
def prepared_baseline(case, data, ids):
path = case/'prepared.json'
prepared = json.loads(private_read(path))
require(prepared['schema_version'] == 1 and all(prepared[k] == value for k,value in ids.items())
and prepared['release_id'] == scope.identifier(data['release_id']), 'Prepared device or release differs')
for k in ('agent_sha256','helper_binary_sha256'):
require(re.fullmatch('[a-f0-9]{64}', prepared[k]) and data.get(k,prepared[k]) == prepared[k],
'Prepared component differs')
journal = scope.identifier(prepared['journal_id'])
decision = scope.identifier(prepared['decision_id'])
checkpoint = prepared['checkpoint']
require(type(checkpoint) is int and checkpoint >= 0, 'Invalid prepared checkpoint')
opened = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
signed = sql(f"""SELECT json_build_object('signed',encode(payload,'hex'),'signature',encode(signature,'hex'))
FROM sync_checkpoints WHERE tenant_id='{ids['tenant_id']}' AND agent_id='{ids['agent_id']}'
AND journal_id='{journal}' AND checkpoint={checkpoint} AND decision_id='{decision}'
AND created_at<='{opened.isoformat()}'""")
raw = bytes.fromhex(signed['signed'])
key_id, public = public_key()
public.verify(bytes.fromhex(signed['signature']), raw)
baseline = json.loads(raw)
require(baseline['domain']=='rmm.reconciliation-decision' and baseline['schema_version']==1
and baseline['key_id']==key_id and baseline['ready'] is True and baseline['reason']=='CONSISTENT'
and baseline['helper_journal_id'] is None and baseline['helper_checkpoint']==0
and baseline['enrollment_generation']==baseline['assignment_version']==1
and all(baseline[k]==prepared[k] for k in ('tenant_id','agent_id','asset_id','journal_id','checkpoint','decision_id')),
'Prepared baseline must be the signed decision of the normal device without a helper')
return prepared, opened
def require_native_verification(data, witness):
proof = data.get('native_verification') or {}
require(data.get('witness') == witness and proof.get('verified') is True and proof.get('applied') is False
and all(proof.get(k)==witness[k] for k in ('case_id','helper_id','decision_id'))
and type(proof.get('attempts')) is int and proof['attempts']==len(witness['attempts']),
'Exact witness and positive native archive verification required before detach')
def window_query(window, case, ids, data):
policy = json.loads(private_read(window))
only, = policy['scopes']
hashes = ('helper_binary_sha256','helper_policy_sha256')
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
windows_update_max_runtime_seconds=3600)
for k in hashes:
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
expected_scope[k] = only[k]
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
'Case identity, privileges or original policy differ')
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
require(opened <= cutoff, 'Invalid maintenance horizon')
return observation(ids,opened.isoformat(),cutoff.isoformat())
def server(action, data):
require(action in ('witness','detach'), 'Only witness and detach are supported')
ids = {k:scope.identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id')}
@@ -163,35 +221,29 @@ def server(action, data):
mode = case.lstat()
require(case.resolve() == case and stat.S_ISDIR(mode.st_mode) and mode.st_uid == 0
and stat.S_IMODE(mode.st_mode) == 0o700, 'Unsafe development window directory')
if not window.exists():
# Even a failed opening can already have registered a real helper journal.
if not window.exists() and not (case/'prepared.json').exists():
# The operator persists its local case ID before prepare. A crash in
# that gap has no authority to detach any helper or alter old jobs.
require(action == 'witness' and data.get('native_window_absent') is True
and private_read(case/'closed') == b'closed\n', 'Unprepared case needs a closed, empty native window')
original = case/'original.json'
require(not original.exists() or private_read(original)==current, 'Policy changed during incomplete preparation')
row = sql(f"""SELECT json_build_object('empty',EXISTS (
SELECT 1 FROM agent_installations i LEFT JOIN agent_sync_state s USING(tenant_id,agent_id)
SELECT 1 FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
WHERE i.tenant_id='{ids['tenant_id']}' AND i.agent_id='{ids['agent_id']}' AND i.asset_id='{ids['asset_id']}'
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND coalesce(s.helper_checkpoint,0)=0
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND s.helper_checkpoint=0
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND
(j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')
OR convert_from(j.payload,'UTF8')::jsonb->>'execution_identity'='system'))))""")
require(row['empty'] and action == 'witness', 'A helper or SYSTEM history exists without an opened scope')
j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))))""")
require(row['empty'], 'Unprepared case still has a helper or an unfinished job')
return None
policy = json.loads(private_read(window))
require(private_read(case/'closed') == b'closed\n' and private_read(case/'original.json') == current,
'Expected this closed scope and its original policy')
only, = policy['scopes']
hashes = ('helper_binary_sha256','helper_policy_sha256')
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
windows_update_max_runtime_seconds=3600)
for k in hashes:
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
expected_scope[k] = only[k]
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
'Case identity, privileges or original policy differ')
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
require(opened <= cutoff, 'Invalid maintenance horizon')
query = observation(ids,opened.isoformat(),cutoff.isoformat())
prepared = None
if window.exists():
query = window_query(window, case, ids, data)
else:
prepared, opened = prepared_baseline(case, data, ids)
query = observation(ids, opened.isoformat())
record = sql(query)
require(record['open_jobs'] == 0, 'Open jobs require separate reconciliation')
saved = case/'detach-intent.json'
@@ -200,16 +252,28 @@ def server(action, data):
if done.exists():
require(receipt and json.loads(private_read(done)) == receipt, 'Completion receipt differs from detach intent')
return receipt
if prepared:
require(record['journal_id']==prepared['journal_id'] and record['checkpoint']>=prepared['checkpoint']
and not record['jobs'], 'Prepared journal changed or a SYSTEM job or grant was created after preparation')
if record['helper_id'] is not None:
require(record['decision_id'] != prepared['decision_id'] and scope.stamp(record['head_created_at'])>=opened,
'The current helper was not bound after preparation')
if record['helper_id'] is None:
if prepared and receipt is None and action=='witness' and record['helper_checkpoint']==0:
return None
require(receipt and receipt['journal_id'] == record['journal_id'] and record['helper_checkpoint'] == 0,
'Missing evidence of the original detach transaction')
if action == 'detach':
if prepared:
require_native_verification(data, receipt)
scope.atomic(done,scope.encode(receipt))
return receipt
witness = verify(record,ids)
require(receipt in (None,witness) and data.get('witness',witness) == witness, 'Device or witness changed')
if action == 'witness':
return witness
if prepared:
require_native_verification(data, witness)
if receipt is None:
scope.atomic(saved,scope.encode(witness))
# Fence a concurrent signed sync (same lock order as the gateway). The server-owned
+51 -25
View File
@@ -74,6 +74,9 @@ def read_registry(data):
AND certificate_sha256=last_authenticated_certificate_sha256 AND last_contact_at > now() - interval '2 minutes',
'enrollment_generation',enrollment_generation,'assignment_version',assignment_version,'capabilities',capabilities)
FROM agent_installations WHERE tenant_id='{tenant}' AND asset_id='{asset}' AND agent_id='{agent}' AND revoked_at IS NULL),
'sync_head',(SELECT json_build_object('journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint) FROM agent_sync_state s
WHERE tenant_id='{tenant}' AND agent_id='{agent}'),
'open_jobs',(SELECT count(*) FROM jobs WHERE tenant_id='{tenant}' AND agent_id='{agent}'
AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
'principal_authorized',EXISTS (SELECT 1 FROM memberships WHERE tenant_id='{tenant}' AND principal='simulated-testprincipal-a'
@@ -156,31 +159,49 @@ def server(action, data):
if not (history/'closed').exists():
atomic(history/'closed',b'closed\n')
return {'central_scope_closed':True}
require(action=='enable', 'Unsupported central operation')
for key in ('helper_binary_sha256','helper_policy_sha256'):
require(re.fullmatch('[a-f0-9]{64}',data[key]),'Invalid helper hash')
require(scoped(data, read_registry(data)), 'Device has no current authenticated narrow helper capabilities')
require(action in ('prepare','enable'), 'Unsupported central operation')
ids={k:identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id','release_id')}
hashes={k:data[k] for k in ('agent_sha256','helper_binary_sha256')}
require(all(re.fullmatch('[a-f0-9]{64}',v) for v in hashes.values()), 'Invalid signed image pair')
expires=stamp(data['expires_at'])
require(utc()<expires<=utc()+timedelta(minutes=120), 'Window expired or exceeds 120 minutes')
if history.exists():
require(not (history/'closed').exists() and (history/'original.json').is_file()
and (history/'window.json').is_file(), 'Case is closed or incomplete')
expected=(history/'window.json').read_bytes()
require(current in (expected,(history/'original.json').read_bytes()), 'Central policy changed while opening')
require(json.loads(expected)['expires_at']==data['expires_at'], 'Cannot extend an existing window')
else:
original,prepared,candidate=history/'original.json',history/'prepared.json',history/'window.json'
require(not (history/'closed').exists(),'Case is already closed')
observation=read_registry(data)
if action=='prepare':
require(usable(observation),'An idle, synchronized and authenticated enrollment is required')
head=observation['sync_head']
require(head and head['helper_id'] is None and head['helper_checkpoint']==0,'Existing helper journal must close first')
old=json.loads(current)
require(old['profile']=='development' and (stamp(old['expires_at'])<=utc() or not old['scopes']),
'Another live maintenance window is active')
scope={k:identifier(data[k]) for k in ('tenant_id','asset_id','agent_id')}
scope.update(enrollment_generation=1,assignment_version=1,principals=['simulated-testprincipal-a'],
acceptance_evidence_id=case,allowed_services=[],max_runtime_seconds=30,
windows_update_max_runtime_seconds=3600,
**{k:data[k] for k in ('helper_binary_sha256','helper_policy_sha256')})
expected=encode(dict(schema_version=3,profile='development',expires_at=data['expires_at'],scopes=[scope]))
history.mkdir(mode=0o700)
atomic(history/'original.json',current)
atomic(history/'window.json',expected)
entry={'schema_version':1,**ids,**hashes,**{k:head[k] for k in ('journal_id','checkpoint','decision_id')}}
history.mkdir(mode=0o700,exist_ok=True)
require(not candidate.exists() and (not original.exists() or original.read_bytes()==current),
'Another policy or opening changed this case')
if prepared.exists():
prior=json.loads(prepared.read_bytes())
require(all(prior[k]==entry[k] for k in (*ids,*hashes,'journal_id')), 'Different prepared enrollment')
else:
atomic(original,current)
atomic(prepared,encode(entry))
return {'prepared':True,'case_id':case,'journal_id':head['journal_id']}
saved=original.read_bytes()
prior=json.loads(prepared.read_bytes())
require(all(prior[k]==v for k,v in (ids|hashes).items()),'Installed pair differs from the prepared case')
require(re.fullmatch('[a-f0-9]{64}',data['helper_policy_sha256']),'Invalid helper policy hash')
require(scoped(data,observation),'Device has no current authenticated narrow helper capabilities')
scope={k:ids[k] for k in ('tenant_id','asset_id','agent_id')}
scope.update(enrollment_generation=1,assignment_version=1,principals=['simulated-testprincipal-a'],
acceptance_evidence_id=case,allowed_services=[],max_runtime_seconds=30,
windows_update_max_runtime_seconds=3600,helper_binary_sha256=hashes['helper_binary_sha256'],
helper_policy_sha256=data['helper_policy_sha256'])
expected=encode(dict(schema_version=3,profile='development',expires_at=data['expires_at'],scopes=[scope]))
if candidate.exists():
require(candidate.read_bytes()==expected and current in (expected,saved),'Changed window or central policy')
else:
require(current==saved,'Central policy changed after preparation')
atomic(candidate,expected)
atomic(path,expected,(st.st_uid,st.st_gid))
return {'central_scope_open':True,'expires_at':data['expires_at']}
@@ -255,9 +276,11 @@ def local():
emit(plan);return
state_dir.mkdir(parents=True,mode=0o700,exist_ok=True)
casefile=state_dir/(data['case_id']+'.json')
with casefile.open('xb') as dst:
data.update(agent_sha256=plan['agent_sha256'],helper_binary_sha256=plan['helper_sha256'])
with os.fdopen(os.open(casefile,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600),'wb') as dst:
dst.write(encode(data));dst.flush();os.fsync(dst.fileno())
emit({'preparing_window':data['case_id'],'expires_at':data['expires_at']})
emit(remote_server('prepare',data))
try:
result=guest('Install',data)
require(result['case_id']==data['case_id'] and result['agent_id']==data['agent_id']
@@ -299,16 +322,19 @@ def close(data):
wait_for(data,lambda row:row['install'] and row['open_jobs']==0,180,
'Attempt still active. Central scope is revoked; retry close when the result is terminal.')
observed=windows('Inspect-DevelopmentWindowsUpdate.ps1',{})
if observed.get('case_id') is not None or observed.get('helper') is not None:
absent=observed.get('case_id') is None and observed.get('helper') is None
if not absent:
require(observed.get('case_id')==data['case_id'],'Local helper belongs to another maintenance window')
if observed.get('release_id')==data['release_id']:
result=guest('Remove',data)
require(result['closed'] and not result['helper_registered'] and result['actor_running'],
'Could not withdraw the helper from this enrollment')
emit(result)
witness=detach_server('witness',data)
witness=detach_server('witness',{**data,'native_window_absent':absent})
if witness and witness.get('helper_id'):
emit(retire_native('Verify',data,witness))
emit(detach_server('detach',{**data,'witness':witness}))
verified=retire_native('Verify',data,witness)
emit(verified)
emit(detach_server('detach',{**data,'witness':witness,'native_verification':verified}))
emit(retire_native('Apply',data,witness))
latest=windows('Inspect-DevelopmentWindowsUpdate.ps1',{})
if latest['release_id']==data['release_id']: