Complete bounded Windows maintenance setup and audited development teardown
This commit is contained in:
@@ -0,0 +1,250 @@
|
||||
"""Detach one closed development maintenance window from its signed sync head."""
|
||||
import fcntl
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import socket
|
||||
import ssl
|
||||
import stat
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import windows_update_scope as scope
|
||||
|
||||
|
||||
BASE = Path('/opt/rust-discovery/devstate')
|
||||
PRINCIPAL = 'simulated-testprincipal-a'
|
||||
SCRIPTS = {
|
||||
'567d7343eecee57a9232b3549ef7c504260cb26da46436ede61e2f354ff3c874': 30,
|
||||
'0fa5f513a06b6cf902a3f9ac1989987642c181eedd6c305b3912771bd10f2ebf': 3600,
|
||||
}
|
||||
PSQL = ['docker', 'exec', '-i', 'rust-discovery-server-postgres-1', 'psql',
|
||||
'-XqAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm']
|
||||
require = scope.require
|
||||
|
||||
|
||||
def sql(statement):
|
||||
result = subprocess.check_output(PSQL, input=statement.encode(), timeout=15)
|
||||
return json.loads(result)
|
||||
|
||||
|
||||
def private_read(path, owner=0):
|
||||
info = path.lstat()
|
||||
require(stat.S_ISREG(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o600
|
||||
and info.st_uid == owner and info.st_nlink == 1 and info.st_size <= 512 * 1024,
|
||||
'Unexpected maintenance record or access mode')
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def public_key():
|
||||
"""Read the server's own public signing key from its authenticated management API."""
|
||||
tls = ssl.create_default_context(cafile=str(BASE / 'pki/ca.pem'))
|
||||
|
||||
class Loopback(http.client.HTTPSConnection):
|
||||
def connect(self):
|
||||
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', 18443), 5),
|
||||
server_hostname='rmm-dev.internal')
|
||||
|
||||
conn = Loopback('rmm-dev.internal', timeout=5, context=tls)
|
||||
try:
|
||||
token = (BASE / 'operator-a.token').read_text().strip()
|
||||
conn.request('GET', '/api/v1/job-signing-key', headers={'Authorization': 'Bearer ' + token})
|
||||
reply = conn.getresponse()
|
||||
require(reply.status == 200, 'Cannot verify the original signed decision against this server')
|
||||
key = json.loads(reply.read(4097))
|
||||
raw = bytes.fromhex(key['public_key_hex'])
|
||||
require(key['algorithm'] == 'Ed25519' and len(raw) == 32
|
||||
and key['key_id'] == hashlib.sha256(raw).hexdigest(), 'Invalid public signing key')
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
||||
return key['key_id'], Ed25519PublicKey.from_public_bytes(raw)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def observation(ids, opened, cutoff):
|
||||
"""Only canonical UUIDs and normalized datetimes reach the query."""
|
||||
tenant, asset, agent = (ids[k] for k in ('tenant_id', 'asset_id', 'agent_id'))
|
||||
return f"""
|
||||
SELECT jsonb_build_object(
|
||||
'open_jobs',(SELECT count(*) FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id
|
||||
AND j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
|
||||
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,
|
||||
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint,
|
||||
'signed',encode(c.payload,'hex'),'signature',encode(c.signature,'hex'),
|
||||
'jobs',COALESCE((SELECT jsonb_agg(jsonb_build_object(
|
||||
'job',j.job_id,'attempt',j.attempt_id,'grant',g.grant_id,'state',j.state,
|
||||
'payload_sha256',j.payload_hash,'bytes_hash',encode(sha256(j.payload),'hex'),
|
||||
'payload_binding',p-'action','action',p->'action',
|
||||
'principal',z.principal,'window_approval',z.created_at BETWEEN '{opened}' AND '{cutoff}',
|
||||
'dev',z.testsimulation AND z.approval_origin='dev_test_browser_simulation',
|
||||
'plan_hash',z.plan_hash,'authorization_id',z.authorization_id,
|
||||
'in_window',g.created_at BETWEEN '{opened}' AND '{cutoff}',
|
||||
'reserved',j.start_reserved,'result_grant',j.result->>'grant_id','revision',j.revision)
|
||||
ORDER BY j.job_id)
|
||||
FROM jobs j JOIN authorizations z USING(tenant_id,operation_id)
|
||||
CROSS JOIN LATERAL (SELECT convert_from(j.payload,'UTF8')::jsonb AS p) jp
|
||||
LEFT JOIN execution_permits g USING(tenant_id,job_id,attempt_id)
|
||||
WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND p->>'execution_identity'='system'
|
||||
AND (z.created_at BETWEEN '{opened}' AND '{cutoff}' OR g.created_at BETWEEN '{opened}' AND '{cutoff}')),
|
||||
'[]'::jsonb)) AS observation
|
||||
FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
|
||||
JOIN sync_checkpoints c USING(tenant_id,agent_id,decision_id)
|
||||
WHERE i.tenant_id='{tenant}' AND i.asset_id='{asset}' AND i.agent_id='{agent}' AND i.revoked_at IS NULL
|
||||
AND i.assignment_version=1 AND i.enrollment_generation=1
|
||||
AND NOT i.sync_recovery_required AND NOT i.sequence_regressed AND NOT i.restore_blocked
|
||||
"""
|
||||
|
||||
|
||||
def verify(snapshot, ids):
|
||||
require(snapshot['open_jobs'] == 0, 'Device has open or uncertain jobs')
|
||||
require(snapshot['helper_id'] and snapshot['helper_checkpoint'] > 0, 'Expected bound helper journal')
|
||||
raw = bytes.fromhex(snapshot['signed'])
|
||||
key_id, public = public_key()
|
||||
public.verify(bytes.fromhex(snapshot['signature']), raw)
|
||||
d = json.loads(raw)
|
||||
fields = ('journal_id', 'checkpoint', 'decision_id', 'helper_checkpoint')
|
||||
require(d['domain'] == 'rmm.reconciliation-decision' and d['schema_version'] == 1
|
||||
and d['key_id'] == key_id and all(d[k] == ids[k] for k in ('tenant_id','asset_id','agent_id'))
|
||||
and d['enrollment_generation'] == d['assignment_version'] == 1
|
||||
and d['ready'] is True and d['helper_ready'] is True and d['reason'] == 'CONSISTENT'
|
||||
and d['helper_journal_id'] == snapshot['helper_id']
|
||||
and all(d[k] == snapshot[k] for k in fields), 'The signed helper decision does not match the current head')
|
||||
outcomes = {a['job_id']: a for a in d['attempts'] if a['execution_identity'] == 'system'}
|
||||
require(len(outcomes) == sum(a['execution_identity'] == 'system' for a in d['attempts']),
|
||||
'Duplicate SYSTEM decision')
|
||||
attempts = []
|
||||
for j in snapshot['jobs']:
|
||||
require(j['grant'], 'A grantless SYSTEM job requires separate reconciliation')
|
||||
require(j['state'] in ('succeeded','failed'), 'A non-definitive SYSTEM result requires separate reconciliation')
|
||||
p = j['payload_binding']
|
||||
act = dict(j['action'])
|
||||
script = hashlib.sha256(act.pop('script_utf8').encode()).hexdigest()
|
||||
require(script in SCRIPTS and 0 < p['max_runtime_seconds'] <= SCRIPTS[script]
|
||||
and act == {'action':'run_script', 'arguments':[], 'expected_stdout':None, 'interpreter':'powershell',
|
||||
'privileged_script_profile':'full_maintenance_v1','windows_execution_profile':'powershell_51_remote_signed'},
|
||||
'Job is outside the two maintenance-window scripts')
|
||||
a = outcomes.pop(j['job'], {})
|
||||
require(j['dev'] and j['principal'] == PRINCIPAL and j['window_approval'] and j['in_window']
|
||||
and j['reserved'] and j['result_grant'] == j['grant'] and j['bytes_hash'] == j['payload_sha256']
|
||||
and all(p[k] == ids[k] for k in ('tenant_id','asset_id','agent_id'))
|
||||
and p['enrollment_generation'] == p['assignment_version'] == 1
|
||||
and p['job_id'] == j['job'] and p['attempt_id'] == j['attempt']
|
||||
and a.get('unlock') is True and a['reason'] == 'DEFINITIVE_OUTCOME_RECONCILED'
|
||||
and a['attempt_id'] == j['attempt'] and a['grant_id'] == j['grant']
|
||||
and a['outcome_state'] == j['state'] and a['revision'] == j['revision']
|
||||
and a['job_payload_hash'] == j['payload_sha256'] and a['authorization_id'] == j['authorization_id']
|
||||
and a['plan_hash'] == j['plan_hash'] and re.fullmatch('[0-9a-f]{64}',a['outcome_sha256'] or ''),
|
||||
'Missing signed evidence of exact consumed grant and terminal result')
|
||||
attempts.append({**{k:j[k] for k in ('job','attempt','grant','state','payload_sha256')},
|
||||
'outcome_sha256':a['outcome_sha256']})
|
||||
require(not outcomes, 'The helper still owns SYSTEM attempts outside this window')
|
||||
return {**ids, **{k:snapshot[k] for k in (*fields,'helper_id')}, 'attempts':attempts}
|
||||
|
||||
|
||||
def server(action, data):
|
||||
require(action in ('witness','detach'), 'Only witness and detach are supported')
|
||||
ids = {k:scope.identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id')}
|
||||
path = scope.policy_location()
|
||||
case = path.parent.parent.parent / ('update-window-' + ids['case_id'])
|
||||
lockfd = os.open(BASE / '.windows-system-scope.lock', os.O_RDWR | os.O_NOFOLLOW)
|
||||
with os.fdopen(lockfd, 'rb') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
current = private_read(path, 10001)
|
||||
current_policy = json.loads(current)
|
||||
require(current_policy['profile'] == 'development' and
|
||||
(not current_policy['scopes'] or scope.stamp(current_policy['expires_at']) <= scope.utc()),
|
||||
'Central development policy must be closed on API and gateway')
|
||||
window = case / 'window.json'
|
||||
if case.exists():
|
||||
mode = case.lstat()
|
||||
require(case.resolve() == case and stat.S_ISDIR(mode.st_mode) and mode.st_uid == 0
|
||||
and stat.S_IMODE(mode.st_mode) == 0o700, 'Unsafe development window directory')
|
||||
if not window.exists():
|
||||
# Even a failed opening can already have registered a real helper journal.
|
||||
row = sql(f"""SELECT json_build_object('empty',EXISTS (
|
||||
SELECT 1 FROM agent_installations i LEFT JOIN agent_sync_state s USING(tenant_id,agent_id)
|
||||
WHERE i.tenant_id='{ids['tenant_id']}' AND i.agent_id='{ids['agent_id']}' AND i.asset_id='{ids['asset_id']}'
|
||||
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND coalesce(s.helper_checkpoint,0)=0
|
||||
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND
|
||||
(j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')
|
||||
OR convert_from(j.payload,'UTF8')::jsonb->>'execution_identity'='system'))))""")
|
||||
require(row['empty'] and action == 'witness', 'A helper or SYSTEM history exists without an opened scope')
|
||||
return None
|
||||
policy = json.loads(private_read(window))
|
||||
require(private_read(case/'closed') == b'closed\n' and private_read(case/'original.json') == current,
|
||||
'Expected this closed scope and its original policy')
|
||||
only, = policy['scopes']
|
||||
hashes = ('helper_binary_sha256','helper_policy_sha256')
|
||||
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
|
||||
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
|
||||
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
|
||||
windows_update_max_runtime_seconds=3600)
|
||||
for k in hashes:
|
||||
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
|
||||
expected_scope[k] = only[k]
|
||||
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
|
||||
'Case identity, privileges or original policy differ')
|
||||
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
|
||||
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
|
||||
require(opened <= cutoff, 'Invalid maintenance horizon')
|
||||
query = observation(ids,opened.isoformat(),cutoff.isoformat())
|
||||
record = sql(query)
|
||||
require(record['open_jobs'] == 0, 'Open jobs require separate reconciliation')
|
||||
saved = case/'detach-intent.json'
|
||||
done = case/'detached.json'
|
||||
receipt = json.loads(private_read(saved)) if saved.exists() else None
|
||||
if done.exists():
|
||||
require(receipt and json.loads(private_read(done)) == receipt, 'Completion receipt differs from detach intent')
|
||||
return receipt
|
||||
if record['helper_id'] is None:
|
||||
require(receipt and receipt['journal_id'] == record['journal_id'] and record['helper_checkpoint'] == 0,
|
||||
'Missing evidence of the original detach transaction')
|
||||
if action == 'detach':
|
||||
scope.atomic(done,scope.encode(receipt))
|
||||
return receipt
|
||||
witness = verify(record,ids)
|
||||
require(receipt in (None,witness) and data.get('witness',witness) == witness, 'Device or witness changed')
|
||||
if action == 'witness':
|
||||
return witness
|
||||
if receipt is None:
|
||||
scope.atomic(saved,scope.encode(witness))
|
||||
# Fence a concurrent signed sync (same lock order as the gateway). The server-owned
|
||||
# approved jobs, outcomes and signed decisions remain immutable throughout detachment.
|
||||
# PostgreSQL jsonb canonical ordering differs from Python: bind our checked snapshot
|
||||
# back as JSONB and compare directly under lock, including jobs, grants and head.
|
||||
observed_hex = scope.encode(record).hex()
|
||||
statements = f"""BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE;
|
||||
SET LOCAL lock_timeout='5s'; SET LOCAL statement_timeout='10s';
|
||||
DO $$ BEGIN PERFORM 1 FROM agent_installations WHERE tenant_id='{ids['tenant_id']}' AND agent_id='{ids['agent_id']}' FOR UPDATE; END $$;
|
||||
WITH checked AS ({query}), changed AS (
|
||||
UPDATE agent_sync_state s SET helper_journal_id=NULL,helper_checkpoint=0
|
||||
FROM checked WHERE s.tenant_id='{ids['tenant_id']}' AND s.agent_id='{ids['agent_id']}'
|
||||
AND s.journal_id='{witness['journal_id']}' AND s.decision_id='{witness['decision_id']}'
|
||||
AND s.checkpoint={witness['checkpoint']} AND s.helper_journal_id='{witness['helper_id']}'
|
||||
AND s.helper_checkpoint={witness['helper_checkpoint']}
|
||||
AND checked.observation = convert_from(decode('{observed_hex}','hex'),'UTF8')::jsonb
|
||||
RETURNING 1)
|
||||
SELECT to_json(count(*)) FROM changed;
|
||||
COMMIT;"""
|
||||
require(sql(statements) == 1, 'Concurrent job or sync transition. Inspect the existing intent before retrying')
|
||||
scope.atomic(done,scope.encode(witness))
|
||||
return witness
|
||||
|
||||
|
||||
def remote_server(action, data):
|
||||
"""Run only the two fixed modules over the existing pinned server SSH transport."""
|
||||
import dev_stage
|
||||
modules = [Path(scope.__file__).read_text(), Path(__file__).read_text()]
|
||||
payload = ('import types,sys,json\n'
|
||||
'scope=types.ModuleType("windows_update_scope");sys.modules[scope.__name__]=scope\n'
|
||||
f'exec({modules[0]!r},scope.__dict__)\n'
|
||||
'detach=types.ModuleType("windows_update_detach")\n'
|
||||
f'exec({modules[1]!r},detach.__dict__)\n'
|
||||
f'print(json.dumps(detach.server({action!r},json.loads({scope.encode(data)!r}))))\n')
|
||||
command = shlex.join(['python3','-I','-'])
|
||||
return json.loads(subprocess.check_output([*dev_stage.ssh_prefix('server'),command],
|
||||
input=payload.encode(),timeout=35))
|
||||
Reference in New Issue
Block a user