Recover a Windows maintenance setup interrupted before policy activation
This commit is contained in:
@@ -65,31 +65,32 @@ def public_key():
|
||||
conn.close()
|
||||
|
||||
|
||||
def observation(ids, opened, cutoff):
|
||||
def observation(ids, opened, cutoff=None):
|
||||
"""Only canonical UUIDs and normalized datetimes reach the query."""
|
||||
tenant, asset, agent = (ids[k] for k in ('tenant_id', 'asset_id', 'agent_id'))
|
||||
horizon = f"BETWEEN '{opened}' AND '{cutoff}'" if cutoff else f">= '{opened}'"
|
||||
return f"""
|
||||
SELECT jsonb_build_object(
|
||||
'open_jobs',(SELECT count(*) FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id
|
||||
AND j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
|
||||
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,
|
||||
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,'head_created_at',c.created_at,
|
||||
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint,
|
||||
'signed',encode(c.payload,'hex'),'signature',encode(c.signature,'hex'),
|
||||
'jobs',COALESCE((SELECT jsonb_agg(jsonb_build_object(
|
||||
'job',j.job_id,'attempt',j.attempt_id,'grant',g.grant_id,'state',j.state,
|
||||
'payload_sha256',j.payload_hash,'bytes_hash',encode(sha256(j.payload),'hex'),
|
||||
'payload_binding',p-'action','action',p->'action',
|
||||
'principal',z.principal,'window_approval',z.created_at BETWEEN '{opened}' AND '{cutoff}',
|
||||
'principal',z.principal,'window_approval',z.created_at {horizon},
|
||||
'dev',z.testsimulation AND z.approval_origin='dev_test_browser_simulation',
|
||||
'plan_hash',z.plan_hash,'authorization_id',z.authorization_id,
|
||||
'in_window',g.created_at BETWEEN '{opened}' AND '{cutoff}',
|
||||
'in_window',g.created_at {horizon},
|
||||
'reserved',j.start_reserved,'result_grant',j.result->>'grant_id','revision',j.revision)
|
||||
ORDER BY j.job_id)
|
||||
FROM jobs j JOIN authorizations z USING(tenant_id,operation_id)
|
||||
CROSS JOIN LATERAL (SELECT convert_from(j.payload,'UTF8')::jsonb AS p) jp
|
||||
LEFT JOIN execution_permits g USING(tenant_id,job_id,attempt_id)
|
||||
WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND p->>'execution_identity'='system'
|
||||
AND (z.created_at BETWEEN '{opened}' AND '{cutoff}' OR g.created_at BETWEEN '{opened}' AND '{cutoff}')),
|
||||
AND (z.created_at {horizon} OR g.created_at {horizon})),
|
||||
'[]'::jsonb)) AS observation
|
||||
FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
|
||||
JOIN sync_checkpoints c USING(tenant_id,agent_id,decision_id)
|
||||
@@ -145,6 +146,63 @@ def verify(snapshot, ids):
|
||||
return {**ids, **{k:snapshot[k] for k in (*fields,'helper_id')}, 'attempts':attempts}
|
||||
|
||||
|
||||
def prepared_baseline(case, data, ids):
|
||||
path = case/'prepared.json'
|
||||
prepared = json.loads(private_read(path))
|
||||
require(prepared['schema_version'] == 1 and all(prepared[k] == value for k,value in ids.items())
|
||||
and prepared['release_id'] == scope.identifier(data['release_id']), 'Prepared device or release differs')
|
||||
for k in ('agent_sha256','helper_binary_sha256'):
|
||||
require(re.fullmatch('[a-f0-9]{64}', prepared[k]) and data.get(k,prepared[k]) == prepared[k],
|
||||
'Prepared component differs')
|
||||
journal = scope.identifier(prepared['journal_id'])
|
||||
decision = scope.identifier(prepared['decision_id'])
|
||||
checkpoint = prepared['checkpoint']
|
||||
require(type(checkpoint) is int and checkpoint >= 0, 'Invalid prepared checkpoint')
|
||||
opened = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
|
||||
signed = sql(f"""SELECT json_build_object('signed',encode(payload,'hex'),'signature',encode(signature,'hex'))
|
||||
FROM sync_checkpoints WHERE tenant_id='{ids['tenant_id']}' AND agent_id='{ids['agent_id']}'
|
||||
AND journal_id='{journal}' AND checkpoint={checkpoint} AND decision_id='{decision}'
|
||||
AND created_at<='{opened.isoformat()}'""")
|
||||
raw = bytes.fromhex(signed['signed'])
|
||||
key_id, public = public_key()
|
||||
public.verify(bytes.fromhex(signed['signature']), raw)
|
||||
baseline = json.loads(raw)
|
||||
require(baseline['domain']=='rmm.reconciliation-decision' and baseline['schema_version']==1
|
||||
and baseline['key_id']==key_id and baseline['ready'] is True and baseline['reason']=='CONSISTENT'
|
||||
and baseline['helper_journal_id'] is None and baseline['helper_checkpoint']==0
|
||||
and baseline['enrollment_generation']==baseline['assignment_version']==1
|
||||
and all(baseline[k]==prepared[k] for k in ('tenant_id','agent_id','asset_id','journal_id','checkpoint','decision_id')),
|
||||
'Prepared baseline must be the signed decision of the normal device without a helper')
|
||||
return prepared, opened
|
||||
|
||||
|
||||
def require_native_verification(data, witness):
|
||||
proof = data.get('native_verification') or {}
|
||||
require(data.get('witness') == witness and proof.get('verified') is True and proof.get('applied') is False
|
||||
and all(proof.get(k)==witness[k] for k in ('case_id','helper_id','decision_id'))
|
||||
and type(proof.get('attempts')) is int and proof['attempts']==len(witness['attempts']),
|
||||
'Exact witness and positive native archive verification required before detach')
|
||||
|
||||
|
||||
def window_query(window, case, ids, data):
|
||||
policy = json.loads(private_read(window))
|
||||
only, = policy['scopes']
|
||||
hashes = ('helper_binary_sha256','helper_policy_sha256')
|
||||
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
|
||||
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
|
||||
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
|
||||
windows_update_max_runtime_seconds=3600)
|
||||
for k in hashes:
|
||||
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
|
||||
expected_scope[k] = only[k]
|
||||
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
|
||||
'Case identity, privileges or original policy differ')
|
||||
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
|
||||
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
|
||||
require(opened <= cutoff, 'Invalid maintenance horizon')
|
||||
return observation(ids,opened.isoformat(),cutoff.isoformat())
|
||||
|
||||
|
||||
def server(action, data):
|
||||
require(action in ('witness','detach'), 'Only witness and detach are supported')
|
||||
ids = {k:scope.identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id')}
|
||||
@@ -163,35 +221,29 @@ def server(action, data):
|
||||
mode = case.lstat()
|
||||
require(case.resolve() == case and stat.S_ISDIR(mode.st_mode) and mode.st_uid == 0
|
||||
and stat.S_IMODE(mode.st_mode) == 0o700, 'Unsafe development window directory')
|
||||
if not window.exists():
|
||||
# Even a failed opening can already have registered a real helper journal.
|
||||
if not window.exists() and not (case/'prepared.json').exists():
|
||||
# The operator persists its local case ID before prepare. A crash in
|
||||
# that gap has no authority to detach any helper or alter old jobs.
|
||||
require(action == 'witness' and data.get('native_window_absent') is True
|
||||
and private_read(case/'closed') == b'closed\n', 'Unprepared case needs a closed, empty native window')
|
||||
original = case/'original.json'
|
||||
require(not original.exists() or private_read(original)==current, 'Policy changed during incomplete preparation')
|
||||
row = sql(f"""SELECT json_build_object('empty',EXISTS (
|
||||
SELECT 1 FROM agent_installations i LEFT JOIN agent_sync_state s USING(tenant_id,agent_id)
|
||||
SELECT 1 FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
|
||||
WHERE i.tenant_id='{ids['tenant_id']}' AND i.agent_id='{ids['agent_id']}' AND i.asset_id='{ids['asset_id']}'
|
||||
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND coalesce(s.helper_checkpoint,0)=0
|
||||
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND s.helper_checkpoint=0
|
||||
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND
|
||||
(j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')
|
||||
OR convert_from(j.payload,'UTF8')::jsonb->>'execution_identity'='system'))))""")
|
||||
require(row['empty'] and action == 'witness', 'A helper or SYSTEM history exists without an opened scope')
|
||||
j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))))""")
|
||||
require(row['empty'], 'Unprepared case still has a helper or an unfinished job')
|
||||
return None
|
||||
policy = json.loads(private_read(window))
|
||||
require(private_read(case/'closed') == b'closed\n' and private_read(case/'original.json') == current,
|
||||
'Expected this closed scope and its original policy')
|
||||
only, = policy['scopes']
|
||||
hashes = ('helper_binary_sha256','helper_policy_sha256')
|
||||
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
|
||||
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
|
||||
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
|
||||
windows_update_max_runtime_seconds=3600)
|
||||
for k in hashes:
|
||||
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
|
||||
expected_scope[k] = only[k]
|
||||
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
|
||||
'Case identity, privileges or original policy differ')
|
||||
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
|
||||
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
|
||||
require(opened <= cutoff, 'Invalid maintenance horizon')
|
||||
query = observation(ids,opened.isoformat(),cutoff.isoformat())
|
||||
prepared = None
|
||||
if window.exists():
|
||||
query = window_query(window, case, ids, data)
|
||||
else:
|
||||
prepared, opened = prepared_baseline(case, data, ids)
|
||||
query = observation(ids, opened.isoformat())
|
||||
record = sql(query)
|
||||
require(record['open_jobs'] == 0, 'Open jobs require separate reconciliation')
|
||||
saved = case/'detach-intent.json'
|
||||
@@ -200,16 +252,28 @@ def server(action, data):
|
||||
if done.exists():
|
||||
require(receipt and json.loads(private_read(done)) == receipt, 'Completion receipt differs from detach intent')
|
||||
return receipt
|
||||
if prepared:
|
||||
require(record['journal_id']==prepared['journal_id'] and record['checkpoint']>=prepared['checkpoint']
|
||||
and not record['jobs'], 'Prepared journal changed or a SYSTEM job or grant was created after preparation')
|
||||
if record['helper_id'] is not None:
|
||||
require(record['decision_id'] != prepared['decision_id'] and scope.stamp(record['head_created_at'])>=opened,
|
||||
'The current helper was not bound after preparation')
|
||||
if record['helper_id'] is None:
|
||||
if prepared and receipt is None and action=='witness' and record['helper_checkpoint']==0:
|
||||
return None
|
||||
require(receipt and receipt['journal_id'] == record['journal_id'] and record['helper_checkpoint'] == 0,
|
||||
'Missing evidence of the original detach transaction')
|
||||
if action == 'detach':
|
||||
if prepared:
|
||||
require_native_verification(data, receipt)
|
||||
scope.atomic(done,scope.encode(receipt))
|
||||
return receipt
|
||||
witness = verify(record,ids)
|
||||
require(receipt in (None,witness) and data.get('witness',witness) == witness, 'Device or witness changed')
|
||||
if action == 'witness':
|
||||
return witness
|
||||
if prepared:
|
||||
require_native_verification(data, witness)
|
||||
if receipt is None:
|
||||
scope.atomic(saved,scope.encode(witness))
|
||||
# Fence a concurrent signed sync (same lock order as the gateway). The server-owned
|
||||
|
||||
@@ -74,6 +74,9 @@ def read_registry(data):
|
||||
AND certificate_sha256=last_authenticated_certificate_sha256 AND last_contact_at > now() - interval '2 minutes',
|
||||
'enrollment_generation',enrollment_generation,'assignment_version',assignment_version,'capabilities',capabilities)
|
||||
FROM agent_installations WHERE tenant_id='{tenant}' AND asset_id='{asset}' AND agent_id='{agent}' AND revoked_at IS NULL),
|
||||
'sync_head',(SELECT json_build_object('journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,
|
||||
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint) FROM agent_sync_state s
|
||||
WHERE tenant_id='{tenant}' AND agent_id='{agent}'),
|
||||
'open_jobs',(SELECT count(*) FROM jobs WHERE tenant_id='{tenant}' AND agent_id='{agent}'
|
||||
AND state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
|
||||
'principal_authorized',EXISTS (SELECT 1 FROM memberships WHERE tenant_id='{tenant}' AND principal='simulated-testprincipal-a'
|
||||
@@ -156,31 +159,49 @@ def server(action, data):
|
||||
if not (history/'closed').exists():
|
||||
atomic(history/'closed',b'closed\n')
|
||||
return {'central_scope_closed':True}
|
||||
require(action=='enable', 'Unsupported central operation')
|
||||
for key in ('helper_binary_sha256','helper_policy_sha256'):
|
||||
require(re.fullmatch('[a-f0-9]{64}',data[key]),'Invalid helper hash')
|
||||
require(scoped(data, read_registry(data)), 'Device has no current authenticated narrow helper capabilities')
|
||||
require(action in ('prepare','enable'), 'Unsupported central operation')
|
||||
ids={k:identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id','release_id')}
|
||||
hashes={k:data[k] for k in ('agent_sha256','helper_binary_sha256')}
|
||||
require(all(re.fullmatch('[a-f0-9]{64}',v) for v in hashes.values()), 'Invalid signed image pair')
|
||||
expires=stamp(data['expires_at'])
|
||||
require(utc()<expires<=utc()+timedelta(minutes=120), 'Window expired or exceeds 120 minutes')
|
||||
if history.exists():
|
||||
require(not (history/'closed').exists() and (history/'original.json').is_file()
|
||||
and (history/'window.json').is_file(), 'Case is closed or incomplete')
|
||||
expected=(history/'window.json').read_bytes()
|
||||
require(current in (expected,(history/'original.json').read_bytes()), 'Central policy changed while opening')
|
||||
require(json.loads(expected)['expires_at']==data['expires_at'], 'Cannot extend an existing window')
|
||||
else:
|
||||
original,prepared,candidate=history/'original.json',history/'prepared.json',history/'window.json'
|
||||
require(not (history/'closed').exists(),'Case is already closed')
|
||||
observation=read_registry(data)
|
||||
if action=='prepare':
|
||||
require(usable(observation),'An idle, synchronized and authenticated enrollment is required')
|
||||
head=observation['sync_head']
|
||||
require(head and head['helper_id'] is None and head['helper_checkpoint']==0,'Existing helper journal must close first')
|
||||
old=json.loads(current)
|
||||
require(old['profile']=='development' and (stamp(old['expires_at'])<=utc() or not old['scopes']),
|
||||
'Another live maintenance window is active')
|
||||
scope={k:identifier(data[k]) for k in ('tenant_id','asset_id','agent_id')}
|
||||
scope.update(enrollment_generation=1,assignment_version=1,principals=['simulated-testprincipal-a'],
|
||||
acceptance_evidence_id=case,allowed_services=[],max_runtime_seconds=30,
|
||||
windows_update_max_runtime_seconds=3600,
|
||||
**{k:data[k] for k in ('helper_binary_sha256','helper_policy_sha256')})
|
||||
expected=encode(dict(schema_version=3,profile='development',expires_at=data['expires_at'],scopes=[scope]))
|
||||
history.mkdir(mode=0o700)
|
||||
atomic(history/'original.json',current)
|
||||
atomic(history/'window.json',expected)
|
||||
entry={'schema_version':1,**ids,**hashes,**{k:head[k] for k in ('journal_id','checkpoint','decision_id')}}
|
||||
history.mkdir(mode=0o700,exist_ok=True)
|
||||
require(not candidate.exists() and (not original.exists() or original.read_bytes()==current),
|
||||
'Another policy or opening changed this case')
|
||||
if prepared.exists():
|
||||
prior=json.loads(prepared.read_bytes())
|
||||
require(all(prior[k]==entry[k] for k in (*ids,*hashes,'journal_id')), 'Different prepared enrollment')
|
||||
else:
|
||||
atomic(original,current)
|
||||
atomic(prepared,encode(entry))
|
||||
return {'prepared':True,'case_id':case,'journal_id':head['journal_id']}
|
||||
saved=original.read_bytes()
|
||||
prior=json.loads(prepared.read_bytes())
|
||||
require(all(prior[k]==v for k,v in (ids|hashes).items()),'Installed pair differs from the prepared case')
|
||||
require(re.fullmatch('[a-f0-9]{64}',data['helper_policy_sha256']),'Invalid helper policy hash')
|
||||
require(scoped(data,observation),'Device has no current authenticated narrow helper capabilities')
|
||||
scope={k:ids[k] for k in ('tenant_id','asset_id','agent_id')}
|
||||
scope.update(enrollment_generation=1,assignment_version=1,principals=['simulated-testprincipal-a'],
|
||||
acceptance_evidence_id=case,allowed_services=[],max_runtime_seconds=30,
|
||||
windows_update_max_runtime_seconds=3600,helper_binary_sha256=hashes['helper_binary_sha256'],
|
||||
helper_policy_sha256=data['helper_policy_sha256'])
|
||||
expected=encode(dict(schema_version=3,profile='development',expires_at=data['expires_at'],scopes=[scope]))
|
||||
if candidate.exists():
|
||||
require(candidate.read_bytes()==expected and current in (expected,saved),'Changed window or central policy')
|
||||
else:
|
||||
require(current==saved,'Central policy changed after preparation')
|
||||
atomic(candidate,expected)
|
||||
atomic(path,expected,(st.st_uid,st.st_gid))
|
||||
return {'central_scope_open':True,'expires_at':data['expires_at']}
|
||||
|
||||
@@ -255,9 +276,11 @@ def local():
|
||||
emit(plan);return
|
||||
state_dir.mkdir(parents=True,mode=0o700,exist_ok=True)
|
||||
casefile=state_dir/(data['case_id']+'.json')
|
||||
with casefile.open('xb') as dst:
|
||||
data.update(agent_sha256=plan['agent_sha256'],helper_binary_sha256=plan['helper_sha256'])
|
||||
with os.fdopen(os.open(casefile,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600),'wb') as dst:
|
||||
dst.write(encode(data));dst.flush();os.fsync(dst.fileno())
|
||||
emit({'preparing_window':data['case_id'],'expires_at':data['expires_at']})
|
||||
emit(remote_server('prepare',data))
|
||||
try:
|
||||
result=guest('Install',data)
|
||||
require(result['case_id']==data['case_id'] and result['agent_id']==data['agent_id']
|
||||
@@ -299,16 +322,19 @@ def close(data):
|
||||
wait_for(data,lambda row:row['install'] and row['open_jobs']==0,180,
|
||||
'Attempt still active. Central scope is revoked; retry close when the result is terminal.')
|
||||
observed=windows('Inspect-DevelopmentWindowsUpdate.ps1',{})
|
||||
if observed.get('case_id') is not None or observed.get('helper') is not None:
|
||||
absent=observed.get('case_id') is None and observed.get('helper') is None
|
||||
if not absent:
|
||||
require(observed.get('case_id')==data['case_id'],'Local helper belongs to another maintenance window')
|
||||
if observed.get('release_id')==data['release_id']:
|
||||
result=guest('Remove',data)
|
||||
require(result['closed'] and not result['helper_registered'] and result['actor_running'],
|
||||
'Could not withdraw the helper from this enrollment')
|
||||
emit(result)
|
||||
witness=detach_server('witness',data)
|
||||
witness=detach_server('witness',{**data,'native_window_absent':absent})
|
||||
if witness and witness.get('helper_id'):
|
||||
emit(retire_native('Verify',data,witness))
|
||||
emit(detach_server('detach',{**data,'witness':witness}))
|
||||
verified=retire_native('Verify',data,witness)
|
||||
emit(verified)
|
||||
emit(detach_server('detach',{**data,'witness':witness,'native_verification':verified}))
|
||||
emit(retire_native('Apply',data,witness))
|
||||
latest=windows('Inspect-DevelopmentWindowsUpdate.ps1',{})
|
||||
if latest['release_id']==data['release_id']:
|
||||
|
||||
Reference in New Issue
Block a user