Files
rust-discovery/tools/windows_update_detach.py
T

325 lines
20 KiB
Python

"""Detach one closed development maintenance window from its signed sync head."""
import fcntl
import hashlib
import http.client
import json
import os
from pathlib import Path
import re
import shlex
import socket
import ssl
import stat
import subprocess
from datetime import datetime, timezone
import windows_update_scope as scope
BASE = Path('/opt/rust-discovery/devstate')
PRINCIPAL = 'simulated-testprincipal-a'
SCRIPTS = {
'567d7343eecee57a9232b3549ef7c504260cb26da46436ede61e2f354ff3c874': 30,
'0fa5f513a06b6cf902a3f9ac1989987642c181eedd6c305b3912771bd10f2ebf': 3600,
}
PSQL = ['docker', 'exec', '-i', 'rust-discovery-server-postgres-1', 'psql',
'-XqAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm']
require = scope.require
def sql(statement):
result = subprocess.check_output(PSQL, input=statement.encode(), timeout=15)
return json.loads(result)
def private_read(path, owner=0):
info = path.lstat()
require(stat.S_ISREG(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o600
and info.st_uid == owner and info.st_nlink == 1 and info.st_size <= 512 * 1024,
'Unexpected maintenance record or access mode')
return path.read_bytes()
def public_key():
"""Read the server's own public signing key from its authenticated management API."""
tls = ssl.create_default_context(cafile=str(BASE / 'pki/ca.pem'))
class Loopback(http.client.HTTPSConnection):
def connect(self):
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', 18443), 5),
server_hostname='rmm-dev.internal')
conn = Loopback('rmm-dev.internal', timeout=5, context=tls)
try:
token = (BASE / 'operator-a.token').read_text().strip()
conn.request('GET', '/api/v1/job-signing-key', headers={'Authorization': 'Bearer ' + token})
reply = conn.getresponse()
require(reply.status == 200, 'Cannot verify the original signed decision against this server')
key = json.loads(reply.read(4097))
raw = bytes.fromhex(key['public_key_hex'])
require(key['algorithm'] == 'Ed25519' and len(raw) == 32
and key['key_id'] == hashlib.sha256(raw).hexdigest(), 'Invalid public signing key')
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
return key['key_id'], Ed25519PublicKey.from_public_bytes(raw)
finally:
conn.close()
def observation(ids, opened, cutoff=None):
"""Only canonical UUIDs and normalized datetimes reach the query."""
tenant, asset, agent = (ids[k] for k in ('tenant_id', 'asset_id', 'agent_id'))
horizon = f"BETWEEN '{opened}' AND '{cutoff}'" if cutoff else f">= '{opened}'"
return f"""
SELECT jsonb_build_object(
'open_jobs',(SELECT count(*) FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id
AND j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,'head_created_at',c.created_at,
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint,
'signed',encode(c.payload,'hex'),'signature',encode(c.signature,'hex'),
'jobs',COALESCE((SELECT jsonb_agg(jsonb_build_object(
'job',j.job_id,'attempt',j.attempt_id,'grant',g.grant_id,'state',j.state,
'payload_sha256',j.payload_hash,'bytes_hash',encode(sha256(j.payload),'hex'),
'payload_binding',p-'action','action',p->'action',
'principal',z.principal,'window_approval',z.created_at {horizon},
'dev',z.testsimulation AND z.approval_origin='dev_test_browser_simulation',
'plan_hash',z.plan_hash,'authorization_id',z.authorization_id,
'in_window',g.created_at {horizon},
'reserved',j.start_reserved,'result_grant',j.result->>'grant_id','revision',j.revision)
ORDER BY j.job_id)
FROM jobs j JOIN authorizations z USING(tenant_id,operation_id)
CROSS JOIN LATERAL (SELECT convert_from(j.payload,'UTF8')::jsonb AS p) jp
LEFT JOIN execution_permits g USING(tenant_id,job_id,attempt_id)
WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND p->>'execution_identity'='system'
AND (z.created_at {horizon} OR g.created_at {horizon})),
'[]'::jsonb)) AS observation
FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
JOIN sync_checkpoints c USING(tenant_id,agent_id,decision_id)
WHERE i.tenant_id='{tenant}' AND i.asset_id='{asset}' AND i.agent_id='{agent}' AND i.revoked_at IS NULL
AND i.assignment_version=1 AND i.enrollment_generation=1
AND NOT i.sync_recovery_required AND NOT i.sequence_regressed AND NOT i.restore_blocked
"""
def verify(snapshot, ids, *, prepared_only=False):
require(snapshot['open_jobs'] == 0, 'Device has open or uncertain jobs')
require(snapshot['helper_id'] and snapshot['helper_checkpoint'] > 0, 'Expected bound helper journal')
raw = bytes.fromhex(snapshot['signed'])
key_id, public = public_key()
public.verify(bytes.fromhex(snapshot['signature']), raw)
d = json.loads(raw)
fields = ('journal_id', 'checkpoint', 'decision_id', 'helper_checkpoint')
require(d['domain'] == 'rmm.reconciliation-decision' and d['schema_version'] == 1
and d['key_id'] == key_id and all(d[k] == ids[k] for k in ('tenant_id','asset_id','agent_id'))
and d['enrollment_generation'] == d['assignment_version'] == 1
and d['ready'] is True and d['helper_ready'] is True and d['reason'] == 'CONSISTENT'
and d['helper_journal_id'] == snapshot['helper_id']
and all(d[k] == snapshot[k] for k in fields), 'The signed helper decision does not match the current head')
outcomes = {a['job_id']: a for a in d['attempts'] if a['execution_identity'] == 'system'}
require(len(outcomes) == sum(a['execution_identity'] == 'system' for a in d['attempts']),
'Duplicate SYSTEM decision')
attempts = []
for j in snapshot['jobs']:
require(j['grant'], 'A grantless SYSTEM job requires separate reconciliation')
require(j['state'] in ('succeeded','failed'), 'A non-definitive SYSTEM result requires separate reconciliation')
p = j['payload_binding']
act = dict(j['action'])
script = hashlib.sha256(act.pop('script_utf8').encode()).hexdigest()
require(script in SCRIPTS and 0 < p['max_runtime_seconds'] <= SCRIPTS[script]
and act == {'action':'run_script', 'arguments':[], 'expected_stdout':None, 'interpreter':'powershell',
'privileged_script_profile':'full_maintenance_v1','windows_execution_profile':'powershell_51_remote_signed'},
'Job is outside the two maintenance-window scripts')
a = outcomes.pop(j['job'], {})
require(j['dev'] and j['principal'] == PRINCIPAL and j['window_approval'] and j['in_window']
and j['reserved'] and j['result_grant'] == j['grant'] and j['bytes_hash'] == j['payload_sha256']
and all(p[k] == ids[k] for k in ('tenant_id','asset_id','agent_id'))
and p['enrollment_generation'] == p['assignment_version'] == 1
and p['job_id'] == j['job'] and p['attempt_id'] == j['attempt']
and a.get('unlock') is True and a['reason'] == 'DEFINITIVE_OUTCOME_RECONCILED'
and a['attempt_id'] == j['attempt'] and a['grant_id'] == j['grant']
and a['outcome_state'] == j['state'] and a['revision'] == j['revision']
and a['job_payload_hash'] == j['payload_sha256'] and a['authorization_id'] == j['authorization_id']
and a['plan_hash'] == j['plan_hash'] and re.fullmatch('[0-9a-f]{64}',a['outcome_sha256'] or ''),
'Missing signed evidence of exact consumed grant and terminal result')
attempts.append({**{k:j[k] for k in ('job','attempt','grant','state','payload_sha256')},
'outcome_sha256':a['outcome_sha256']})
require(not outcomes, 'The helper still owns SYSTEM attempts outside this window')
witness = {**ids, **{k:snapshot[k] for k in (*fields,'helper_id')}, 'attempts':attempts}
if prepared_only and not attempts and d['helper_checkpoint'] == 1:
from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
# Binding is hashed in Rust's fixed declaration order.
binding = {k:ids[k] for k in ('tenant_id','asset_id','agent_id')}
binding.update(enrollment_generation=1,assignment_version=1,
public_key_hex=public.public_bytes(Encoding.Raw,PublicFormat.Raw).hex())
witness['prepared_first_capture'] = {
'binding_sha256':hashlib.sha256(json.dumps(binding,separators=(',',':')).encode()).hexdigest(),
'request_id':d['request_id'], 'request_hash':d['request_hash'], 'boot_id':d['boot_id']}
return witness
def prepared_baseline(case, data, ids):
path = case/'prepared.json'
prepared = json.loads(private_read(path))
require(prepared['schema_version'] == 1 and all(prepared[k] == value for k,value in ids.items())
and prepared['release_id'] == scope.identifier(data['release_id']), 'Prepared device or release differs')
for k in ('agent_sha256','helper_binary_sha256'):
require(re.fullmatch('[a-f0-9]{64}', prepared[k]) and data.get(k,prepared[k]) == prepared[k],
'Prepared component differs')
journal = scope.identifier(prepared['journal_id'])
decision = scope.identifier(prepared['decision_id'])
checkpoint = prepared['checkpoint']
require(type(checkpoint) is int and checkpoint >= 0, 'Invalid prepared checkpoint')
opened = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
signed = sql(f"""SELECT json_build_object('signed',encode(payload,'hex'),'signature',encode(signature,'hex'))
FROM sync_checkpoints WHERE tenant_id='{ids['tenant_id']}' AND agent_id='{ids['agent_id']}'
AND journal_id='{journal}' AND checkpoint={checkpoint} AND decision_id='{decision}'
AND created_at<='{opened.isoformat()}'""")
raw = bytes.fromhex(signed['signed'])
key_id, public = public_key()
public.verify(bytes.fromhex(signed['signature']), raw)
baseline = json.loads(raw)
require(baseline['domain']=='rmm.reconciliation-decision' and baseline['schema_version']==1
and baseline['key_id']==key_id and baseline['ready'] is True and baseline['reason']=='CONSISTENT'
and baseline['helper_journal_id'] is None and baseline['helper_checkpoint']==0
and baseline['enrollment_generation']==baseline['assignment_version']==1
and all(baseline[k]==prepared[k] for k in ('tenant_id','agent_id','asset_id','journal_id','checkpoint','decision_id')),
'Prepared baseline must be the signed decision of the normal device without a helper')
return prepared, opened
def require_native_verification(data, witness):
proof = data.get('native_verification') or {}
require(data.get('witness') == witness and proof.get('verified') is True and proof.get('applied') is False
and all(proof.get(k)==witness[k] for k in ('case_id','helper_id','decision_id'))
and type(proof.get('attempts')) is int and proof['attempts']==len(witness['attempts']),
'Exact witness and positive native archive verification required before detach')
def window_query(window, case, ids, data):
policy = json.loads(private_read(window))
only, = policy['scopes']
hashes = ('helper_binary_sha256','helper_policy_sha256')
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
windows_update_max_runtime_seconds=3600)
for k in hashes:
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
expected_scope[k] = only[k]
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
'Case identity, privileges or original policy differ')
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
require(opened <= cutoff, 'Invalid maintenance horizon')
return observation(ids,opened.isoformat(),cutoff.isoformat())
def server(action, data):
require(action in ('witness','detach'), 'Only witness and detach are supported')
ids = {k:scope.identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id')}
path = scope.policy_location()
case = path.parent.parent.parent / ('update-window-' + ids['case_id'])
lockfd = os.open(BASE / '.windows-system-scope.lock', os.O_RDWR | os.O_NOFOLLOW)
with os.fdopen(lockfd, 'rb') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
current = private_read(path, 10001)
current_policy = json.loads(current)
require(current_policy['profile'] == 'development' and
(not current_policy['scopes'] or scope.stamp(current_policy['expires_at']) <= scope.utc()),
'Central development policy must be closed on API and gateway')
window = case / 'window.json'
if case.exists():
mode = case.lstat()
require(case.resolve() == case and stat.S_ISDIR(mode.st_mode) and mode.st_uid == 0
and stat.S_IMODE(mode.st_mode) == 0o700, 'Unsafe development window directory')
if not window.exists() and not (case/'prepared.json').exists():
# The operator persists its local case ID before prepare. A crash in
# that gap has no authority to detach any helper or alter old jobs.
require(action == 'witness' and data.get('native_window_absent') is True
and private_read(case/'closed') == b'closed\n', 'Unprepared case needs a closed, empty native window')
original = case/'original.json'
require(not original.exists() or private_read(original)==current, 'Policy changed during incomplete preparation')
row = sql(f"""SELECT json_build_object('empty',EXISTS (
SELECT 1 FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
WHERE i.tenant_id='{ids['tenant_id']}' AND i.agent_id='{ids['agent_id']}' AND i.asset_id='{ids['asset_id']}'
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND s.helper_checkpoint=0
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND
j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))))""")
require(row['empty'], 'Unprepared case still has a helper or an unfinished job')
return None
require(private_read(case/'closed') == b'closed\n' and private_read(case/'original.json') == current,
'Expected this closed scope and its original policy')
prepared = None
if window.exists():
query = window_query(window, case, ids, data)
else:
prepared, opened = prepared_baseline(case, data, ids)
query = observation(ids, opened.isoformat())
record = sql(query)
require(record['open_jobs'] == 0, 'Open jobs require separate reconciliation')
saved = case/'detach-intent.json'
done = case/'detached.json'
receipt = json.loads(private_read(saved)) if saved.exists() else None
if done.exists():
require(receipt and json.loads(private_read(done)) == receipt, 'Completion receipt differs from detach intent')
return receipt
if prepared:
require(record['journal_id']==prepared['journal_id'] and record['checkpoint']>=prepared['checkpoint']
and not record['jobs'], 'Prepared journal changed or a SYSTEM job or grant was created after preparation')
if record['helper_id'] is not None:
require(record['decision_id'] != prepared['decision_id'] and scope.stamp(record['head_created_at'])>=opened,
'The current helper was not bound after preparation')
if record['helper_id'] is None:
if prepared and receipt is None and action=='witness' and record['helper_checkpoint']==0:
return None
require(receipt and receipt['journal_id'] == record['journal_id'] and record['helper_checkpoint'] == 0,
'Missing evidence of the original detach transaction')
if action == 'detach':
if prepared:
require_native_verification(data, receipt)
scope.atomic(done,scope.encode(receipt))
return receipt
witness = verify(record,ids,prepared_only=prepared is not None and (receipt is None or 'prepared_first_capture' in receipt))
require(receipt in (None,witness) and data.get('witness',witness) == witness, 'Device or witness changed')
if action == 'witness':
return witness
if prepared:
require_native_verification(data, witness)
if receipt is None:
scope.atomic(saved,scope.encode(witness))
# Fence a concurrent signed sync (same lock order as the gateway). The server-owned
# approved jobs, outcomes and signed decisions remain immutable throughout detachment.
# PostgreSQL jsonb canonical ordering differs from Python: bind our checked snapshot
# back as JSONB and compare directly under lock, including jobs, grants and head.
observed_hex = scope.encode(record).hex()
statements = f"""BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE;
SET LOCAL lock_timeout='5s'; SET LOCAL statement_timeout='10s';
DO $$ BEGIN PERFORM 1 FROM agent_installations WHERE tenant_id='{ids['tenant_id']}' AND agent_id='{ids['agent_id']}' FOR UPDATE; END $$;
WITH checked AS ({query}), changed AS (
UPDATE agent_sync_state s SET helper_journal_id=NULL,helper_checkpoint=0
FROM checked WHERE s.tenant_id='{ids['tenant_id']}' AND s.agent_id='{ids['agent_id']}'
AND s.journal_id='{witness['journal_id']}' AND s.decision_id='{witness['decision_id']}'
AND s.checkpoint={witness['checkpoint']} AND s.helper_journal_id='{witness['helper_id']}'
AND s.helper_checkpoint={witness['helper_checkpoint']}
AND checked.observation = convert_from(decode('{observed_hex}','hex'),'UTF8')::jsonb
RETURNING 1)
SELECT to_json(count(*)) FROM changed;
COMMIT;"""
require(sql(statements) == 1, 'Concurrent job or sync transition. Inspect the existing intent before retrying')
scope.atomic(done,scope.encode(witness))
return witness
def remote_server(action, data):
"""Run only the two fixed modules over the existing pinned server SSH transport."""
import dev_stage
modules = [Path(scope.__file__).read_text(), Path(__file__).read_text()]
payload = ('import types,sys,json\n'
'scope=types.ModuleType("windows_update_scope");sys.modules[scope.__name__]=scope\n'
f'exec({modules[0]!r},scope.__dict__)\n'
'detach=types.ModuleType("windows_update_detach")\n'
f'exec({modules[1]!r},detach.__dict__)\n'
f'print(json.dumps(detach.server({action!r},json.loads({scope.encode(data)!r}))))\n')
command = shlex.join(['python3','-I','-'])
return json.loads(subprocess.check_output([*dev_stage.ssh_prefix('server'),command],
input=payload.encode(),timeout=35))