Recover a Windows maintenance setup interrupted before policy activation
This commit is contained in:
@@ -65,31 +65,32 @@ def public_key():
|
||||
conn.close()
|
||||
|
||||
|
||||
def observation(ids, opened, cutoff):
|
||||
def observation(ids, opened, cutoff=None):
|
||||
"""Only canonical UUIDs and normalized datetimes reach the query."""
|
||||
tenant, asset, agent = (ids[k] for k in ('tenant_id', 'asset_id', 'agent_id'))
|
||||
horizon = f"BETWEEN '{opened}' AND '{cutoff}'" if cutoff else f">= '{opened}'"
|
||||
return f"""
|
||||
SELECT jsonb_build_object(
|
||||
'open_jobs',(SELECT count(*) FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id
|
||||
AND j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
|
||||
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,
|
||||
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,'head_created_at',c.created_at,
|
||||
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint,
|
||||
'signed',encode(c.payload,'hex'),'signature',encode(c.signature,'hex'),
|
||||
'jobs',COALESCE((SELECT jsonb_agg(jsonb_build_object(
|
||||
'job',j.job_id,'attempt',j.attempt_id,'grant',g.grant_id,'state',j.state,
|
||||
'payload_sha256',j.payload_hash,'bytes_hash',encode(sha256(j.payload),'hex'),
|
||||
'payload_binding',p-'action','action',p->'action',
|
||||
'principal',z.principal,'window_approval',z.created_at BETWEEN '{opened}' AND '{cutoff}',
|
||||
'principal',z.principal,'window_approval',z.created_at {horizon},
|
||||
'dev',z.testsimulation AND z.approval_origin='dev_test_browser_simulation',
|
||||
'plan_hash',z.plan_hash,'authorization_id',z.authorization_id,
|
||||
'in_window',g.created_at BETWEEN '{opened}' AND '{cutoff}',
|
||||
'in_window',g.created_at {horizon},
|
||||
'reserved',j.start_reserved,'result_grant',j.result->>'grant_id','revision',j.revision)
|
||||
ORDER BY j.job_id)
|
||||
FROM jobs j JOIN authorizations z USING(tenant_id,operation_id)
|
||||
CROSS JOIN LATERAL (SELECT convert_from(j.payload,'UTF8')::jsonb AS p) jp
|
||||
LEFT JOIN execution_permits g USING(tenant_id,job_id,attempt_id)
|
||||
WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND p->>'execution_identity'='system'
|
||||
AND (z.created_at BETWEEN '{opened}' AND '{cutoff}' OR g.created_at BETWEEN '{opened}' AND '{cutoff}')),
|
||||
AND (z.created_at {horizon} OR g.created_at {horizon})),
|
||||
'[]'::jsonb)) AS observation
|
||||
FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
|
||||
JOIN sync_checkpoints c USING(tenant_id,agent_id,decision_id)
|
||||
@@ -145,6 +146,63 @@ def verify(snapshot, ids):
|
||||
return {**ids, **{k:snapshot[k] for k in (*fields,'helper_id')}, 'attempts':attempts}
|
||||
|
||||
|
||||
def prepared_baseline(case, data, ids):
|
||||
path = case/'prepared.json'
|
||||
prepared = json.loads(private_read(path))
|
||||
require(prepared['schema_version'] == 1 and all(prepared[k] == value for k,value in ids.items())
|
||||
and prepared['release_id'] == scope.identifier(data['release_id']), 'Prepared device or release differs')
|
||||
for k in ('agent_sha256','helper_binary_sha256'):
|
||||
require(re.fullmatch('[a-f0-9]{64}', prepared[k]) and data.get(k,prepared[k]) == prepared[k],
|
||||
'Prepared component differs')
|
||||
journal = scope.identifier(prepared['journal_id'])
|
||||
decision = scope.identifier(prepared['decision_id'])
|
||||
checkpoint = prepared['checkpoint']
|
||||
require(type(checkpoint) is int and checkpoint >= 0, 'Invalid prepared checkpoint')
|
||||
opened = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
|
||||
signed = sql(f"""SELECT json_build_object('signed',encode(payload,'hex'),'signature',encode(signature,'hex'))
|
||||
FROM sync_checkpoints WHERE tenant_id='{ids['tenant_id']}' AND agent_id='{ids['agent_id']}'
|
||||
AND journal_id='{journal}' AND checkpoint={checkpoint} AND decision_id='{decision}'
|
||||
AND created_at<='{opened.isoformat()}'""")
|
||||
raw = bytes.fromhex(signed['signed'])
|
||||
key_id, public = public_key()
|
||||
public.verify(bytes.fromhex(signed['signature']), raw)
|
||||
baseline = json.loads(raw)
|
||||
require(baseline['domain']=='rmm.reconciliation-decision' and baseline['schema_version']==1
|
||||
and baseline['key_id']==key_id and baseline['ready'] is True and baseline['reason']=='CONSISTENT'
|
||||
and baseline['helper_journal_id'] is None and baseline['helper_checkpoint']==0
|
||||
and baseline['enrollment_generation']==baseline['assignment_version']==1
|
||||
and all(baseline[k]==prepared[k] for k in ('tenant_id','agent_id','asset_id','journal_id','checkpoint','decision_id')),
|
||||
'Prepared baseline must be the signed decision of the normal device without a helper')
|
||||
return prepared, opened
|
||||
|
||||
|
||||
def require_native_verification(data, witness):
|
||||
proof = data.get('native_verification') or {}
|
||||
require(data.get('witness') == witness and proof.get('verified') is True and proof.get('applied') is False
|
||||
and all(proof.get(k)==witness[k] for k in ('case_id','helper_id','decision_id'))
|
||||
and type(proof.get('attempts')) is int and proof['attempts']==len(witness['attempts']),
|
||||
'Exact witness and positive native archive verification required before detach')
|
||||
|
||||
|
||||
def window_query(window, case, ids, data):
|
||||
policy = json.loads(private_read(window))
|
||||
only, = policy['scopes']
|
||||
hashes = ('helper_binary_sha256','helper_policy_sha256')
|
||||
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
|
||||
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
|
||||
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
|
||||
windows_update_max_runtime_seconds=3600)
|
||||
for k in hashes:
|
||||
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
|
||||
expected_scope[k] = only[k]
|
||||
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
|
||||
'Case identity, privileges or original policy differ')
|
||||
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
|
||||
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
|
||||
require(opened <= cutoff, 'Invalid maintenance horizon')
|
||||
return observation(ids,opened.isoformat(),cutoff.isoformat())
|
||||
|
||||
|
||||
def server(action, data):
|
||||
require(action in ('witness','detach'), 'Only witness and detach are supported')
|
||||
ids = {k:scope.identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id')}
|
||||
@@ -163,35 +221,29 @@ def server(action, data):
|
||||
mode = case.lstat()
|
||||
require(case.resolve() == case and stat.S_ISDIR(mode.st_mode) and mode.st_uid == 0
|
||||
and stat.S_IMODE(mode.st_mode) == 0o700, 'Unsafe development window directory')
|
||||
if not window.exists():
|
||||
# Even a failed opening can already have registered a real helper journal.
|
||||
if not window.exists() and not (case/'prepared.json').exists():
|
||||
# The operator persists its local case ID before prepare. A crash in
|
||||
# that gap has no authority to detach any helper or alter old jobs.
|
||||
require(action == 'witness' and data.get('native_window_absent') is True
|
||||
and private_read(case/'closed') == b'closed\n', 'Unprepared case needs a closed, empty native window')
|
||||
original = case/'original.json'
|
||||
require(not original.exists() or private_read(original)==current, 'Policy changed during incomplete preparation')
|
||||
row = sql(f"""SELECT json_build_object('empty',EXISTS (
|
||||
SELECT 1 FROM agent_installations i LEFT JOIN agent_sync_state s USING(tenant_id,agent_id)
|
||||
SELECT 1 FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
|
||||
WHERE i.tenant_id='{ids['tenant_id']}' AND i.agent_id='{ids['agent_id']}' AND i.asset_id='{ids['asset_id']}'
|
||||
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND coalesce(s.helper_checkpoint,0)=0
|
||||
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND s.helper_checkpoint=0
|
||||
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND
|
||||
(j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')
|
||||
OR convert_from(j.payload,'UTF8')::jsonb->>'execution_identity'='system'))))""")
|
||||
require(row['empty'] and action == 'witness', 'A helper or SYSTEM history exists without an opened scope')
|
||||
j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))))""")
|
||||
require(row['empty'], 'Unprepared case still has a helper or an unfinished job')
|
||||
return None
|
||||
policy = json.loads(private_read(window))
|
||||
require(private_read(case/'closed') == b'closed\n' and private_read(case/'original.json') == current,
|
||||
'Expected this closed scope and its original policy')
|
||||
only, = policy['scopes']
|
||||
hashes = ('helper_binary_sha256','helper_policy_sha256')
|
||||
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
|
||||
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
|
||||
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
|
||||
windows_update_max_runtime_seconds=3600)
|
||||
for k in hashes:
|
||||
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
|
||||
expected_scope[k] = only[k]
|
||||
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
|
||||
'Case identity, privileges or original policy differ')
|
||||
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
|
||||
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
|
||||
require(opened <= cutoff, 'Invalid maintenance horizon')
|
||||
query = observation(ids,opened.isoformat(),cutoff.isoformat())
|
||||
prepared = None
|
||||
if window.exists():
|
||||
query = window_query(window, case, ids, data)
|
||||
else:
|
||||
prepared, opened = prepared_baseline(case, data, ids)
|
||||
query = observation(ids, opened.isoformat())
|
||||
record = sql(query)
|
||||
require(record['open_jobs'] == 0, 'Open jobs require separate reconciliation')
|
||||
saved = case/'detach-intent.json'
|
||||
@@ -200,16 +252,28 @@ def server(action, data):
|
||||
if done.exists():
|
||||
require(receipt and json.loads(private_read(done)) == receipt, 'Completion receipt differs from detach intent')
|
||||
return receipt
|
||||
if prepared:
|
||||
require(record['journal_id']==prepared['journal_id'] and record['checkpoint']>=prepared['checkpoint']
|
||||
and not record['jobs'], 'Prepared journal changed or a SYSTEM job or grant was created after preparation')
|
||||
if record['helper_id'] is not None:
|
||||
require(record['decision_id'] != prepared['decision_id'] and scope.stamp(record['head_created_at'])>=opened,
|
||||
'The current helper was not bound after preparation')
|
||||
if record['helper_id'] is None:
|
||||
if prepared and receipt is None and action=='witness' and record['helper_checkpoint']==0:
|
||||
return None
|
||||
require(receipt and receipt['journal_id'] == record['journal_id'] and record['helper_checkpoint'] == 0,
|
||||
'Missing evidence of the original detach transaction')
|
||||
if action == 'detach':
|
||||
if prepared:
|
||||
require_native_verification(data, receipt)
|
||||
scope.atomic(done,scope.encode(receipt))
|
||||
return receipt
|
||||
witness = verify(record,ids)
|
||||
require(receipt in (None,witness) and data.get('witness',witness) == witness, 'Device or witness changed')
|
||||
if action == 'witness':
|
||||
return witness
|
||||
if prepared:
|
||||
require_native_verification(data, witness)
|
||||
if receipt is None:
|
||||
scope.atomic(saved,scope.encode(witness))
|
||||
# Fence a concurrent signed sync (same lock order as the gateway). The server-owned
|
||||
|
||||
Reference in New Issue
Block a user