251 lines
15 KiB
Python
251 lines
15 KiB
Python
"""Detach one closed development maintenance window from its signed sync head."""
|
|
import fcntl
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shlex
|
|
import socket
|
|
import ssl
|
|
import stat
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
|
|
import windows_update_scope as scope
|
|
|
|
|
|
BASE = Path('/opt/rust-discovery/devstate')
|
|
PRINCIPAL = 'simulated-testprincipal-a'
|
|
SCRIPTS = {
|
|
'567d7343eecee57a9232b3549ef7c504260cb26da46436ede61e2f354ff3c874': 30,
|
|
'0fa5f513a06b6cf902a3f9ac1989987642c181eedd6c305b3912771bd10f2ebf': 3600,
|
|
}
|
|
PSQL = ['docker', 'exec', '-i', 'rust-discovery-server-postgres-1', 'psql',
|
|
'-XqAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm']
|
|
require = scope.require
|
|
|
|
|
|
def sql(statement):
|
|
result = subprocess.check_output(PSQL, input=statement.encode(), timeout=15)
|
|
return json.loads(result)
|
|
|
|
|
|
def private_read(path, owner=0):
|
|
info = path.lstat()
|
|
require(stat.S_ISREG(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o600
|
|
and info.st_uid == owner and info.st_nlink == 1 and info.st_size <= 512 * 1024,
|
|
'Unexpected maintenance record or access mode')
|
|
return path.read_bytes()
|
|
|
|
|
|
def public_key():
|
|
"""Read the server's own public signing key from its authenticated management API."""
|
|
tls = ssl.create_default_context(cafile=str(BASE / 'pki/ca.pem'))
|
|
|
|
class Loopback(http.client.HTTPSConnection):
|
|
def connect(self):
|
|
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', 18443), 5),
|
|
server_hostname='rmm-dev.internal')
|
|
|
|
conn = Loopback('rmm-dev.internal', timeout=5, context=tls)
|
|
try:
|
|
token = (BASE / 'operator-a.token').read_text().strip()
|
|
conn.request('GET', '/api/v1/job-signing-key', headers={'Authorization': 'Bearer ' + token})
|
|
reply = conn.getresponse()
|
|
require(reply.status == 200, 'Cannot verify the original signed decision against this server')
|
|
key = json.loads(reply.read(4097))
|
|
raw = bytes.fromhex(key['public_key_hex'])
|
|
require(key['algorithm'] == 'Ed25519' and len(raw) == 32
|
|
and key['key_id'] == hashlib.sha256(raw).hexdigest(), 'Invalid public signing key')
|
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
|
return key['key_id'], Ed25519PublicKey.from_public_bytes(raw)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def observation(ids, opened, cutoff):
|
|
"""Only canonical UUIDs and normalized datetimes reach the query."""
|
|
tenant, asset, agent = (ids[k] for k in ('tenant_id', 'asset_id', 'agent_id'))
|
|
return f"""
|
|
SELECT jsonb_build_object(
|
|
'open_jobs',(SELECT count(*) FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id
|
|
AND j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')),
|
|
'journal_id',s.journal_id,'checkpoint',s.checkpoint,'decision_id',s.decision_id,
|
|
'helper_id',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint,
|
|
'signed',encode(c.payload,'hex'),'signature',encode(c.signature,'hex'),
|
|
'jobs',COALESCE((SELECT jsonb_agg(jsonb_build_object(
|
|
'job',j.job_id,'attempt',j.attempt_id,'grant',g.grant_id,'state',j.state,
|
|
'payload_sha256',j.payload_hash,'bytes_hash',encode(sha256(j.payload),'hex'),
|
|
'payload_binding',p-'action','action',p->'action',
|
|
'principal',z.principal,'window_approval',z.created_at BETWEEN '{opened}' AND '{cutoff}',
|
|
'dev',z.testsimulation AND z.approval_origin='dev_test_browser_simulation',
|
|
'plan_hash',z.plan_hash,'authorization_id',z.authorization_id,
|
|
'in_window',g.created_at BETWEEN '{opened}' AND '{cutoff}',
|
|
'reserved',j.start_reserved,'result_grant',j.result->>'grant_id','revision',j.revision)
|
|
ORDER BY j.job_id)
|
|
FROM jobs j JOIN authorizations z USING(tenant_id,operation_id)
|
|
CROSS JOIN LATERAL (SELECT convert_from(j.payload,'UTF8')::jsonb AS p) jp
|
|
LEFT JOIN execution_permits g USING(tenant_id,job_id,attempt_id)
|
|
WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND p->>'execution_identity'='system'
|
|
AND (z.created_at BETWEEN '{opened}' AND '{cutoff}' OR g.created_at BETWEEN '{opened}' AND '{cutoff}')),
|
|
'[]'::jsonb)) AS observation
|
|
FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id)
|
|
JOIN sync_checkpoints c USING(tenant_id,agent_id,decision_id)
|
|
WHERE i.tenant_id='{tenant}' AND i.asset_id='{asset}' AND i.agent_id='{agent}' AND i.revoked_at IS NULL
|
|
AND i.assignment_version=1 AND i.enrollment_generation=1
|
|
AND NOT i.sync_recovery_required AND NOT i.sequence_regressed AND NOT i.restore_blocked
|
|
"""
|
|
|
|
|
|
def verify(snapshot, ids):
|
|
require(snapshot['open_jobs'] == 0, 'Device has open or uncertain jobs')
|
|
require(snapshot['helper_id'] and snapshot['helper_checkpoint'] > 0, 'Expected bound helper journal')
|
|
raw = bytes.fromhex(snapshot['signed'])
|
|
key_id, public = public_key()
|
|
public.verify(bytes.fromhex(snapshot['signature']), raw)
|
|
d = json.loads(raw)
|
|
fields = ('journal_id', 'checkpoint', 'decision_id', 'helper_checkpoint')
|
|
require(d['domain'] == 'rmm.reconciliation-decision' and d['schema_version'] == 1
|
|
and d['key_id'] == key_id and all(d[k] == ids[k] for k in ('tenant_id','asset_id','agent_id'))
|
|
and d['enrollment_generation'] == d['assignment_version'] == 1
|
|
and d['ready'] is True and d['helper_ready'] is True and d['reason'] == 'CONSISTENT'
|
|
and d['helper_journal_id'] == snapshot['helper_id']
|
|
and all(d[k] == snapshot[k] for k in fields), 'The signed helper decision does not match the current head')
|
|
outcomes = {a['job_id']: a for a in d['attempts'] if a['execution_identity'] == 'system'}
|
|
require(len(outcomes) == sum(a['execution_identity'] == 'system' for a in d['attempts']),
|
|
'Duplicate SYSTEM decision')
|
|
attempts = []
|
|
for j in snapshot['jobs']:
|
|
require(j['grant'], 'A grantless SYSTEM job requires separate reconciliation')
|
|
require(j['state'] in ('succeeded','failed'), 'A non-definitive SYSTEM result requires separate reconciliation')
|
|
p = j['payload_binding']
|
|
act = dict(j['action'])
|
|
script = hashlib.sha256(act.pop('script_utf8').encode()).hexdigest()
|
|
require(script in SCRIPTS and 0 < p['max_runtime_seconds'] <= SCRIPTS[script]
|
|
and act == {'action':'run_script', 'arguments':[], 'expected_stdout':None, 'interpreter':'powershell',
|
|
'privileged_script_profile':'full_maintenance_v1','windows_execution_profile':'powershell_51_remote_signed'},
|
|
'Job is outside the two maintenance-window scripts')
|
|
a = outcomes.pop(j['job'], {})
|
|
require(j['dev'] and j['principal'] == PRINCIPAL and j['window_approval'] and j['in_window']
|
|
and j['reserved'] and j['result_grant'] == j['grant'] and j['bytes_hash'] == j['payload_sha256']
|
|
and all(p[k] == ids[k] for k in ('tenant_id','asset_id','agent_id'))
|
|
and p['enrollment_generation'] == p['assignment_version'] == 1
|
|
and p['job_id'] == j['job'] and p['attempt_id'] == j['attempt']
|
|
and a.get('unlock') is True and a['reason'] == 'DEFINITIVE_OUTCOME_RECONCILED'
|
|
and a['attempt_id'] == j['attempt'] and a['grant_id'] == j['grant']
|
|
and a['outcome_state'] == j['state'] and a['revision'] == j['revision']
|
|
and a['job_payload_hash'] == j['payload_sha256'] and a['authorization_id'] == j['authorization_id']
|
|
and a['plan_hash'] == j['plan_hash'] and re.fullmatch('[0-9a-f]{64}',a['outcome_sha256'] or ''),
|
|
'Missing signed evidence of exact consumed grant and terminal result')
|
|
attempts.append({**{k:j[k] for k in ('job','attempt','grant','state','payload_sha256')},
|
|
'outcome_sha256':a['outcome_sha256']})
|
|
require(not outcomes, 'The helper still owns SYSTEM attempts outside this window')
|
|
return {**ids, **{k:snapshot[k] for k in (*fields,'helper_id')}, 'attempts':attempts}
|
|
|
|
|
|
def server(action, data):
|
|
require(action in ('witness','detach'), 'Only witness and detach are supported')
|
|
ids = {k:scope.identifier(data[k]) for k in ('case_id','tenant_id','asset_id','agent_id')}
|
|
path = scope.policy_location()
|
|
case = path.parent.parent.parent / ('update-window-' + ids['case_id'])
|
|
lockfd = os.open(BASE / '.windows-system-scope.lock', os.O_RDWR | os.O_NOFOLLOW)
|
|
with os.fdopen(lockfd, 'rb') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
current = private_read(path, 10001)
|
|
current_policy = json.loads(current)
|
|
require(current_policy['profile'] == 'development' and
|
|
(not current_policy['scopes'] or scope.stamp(current_policy['expires_at']) <= scope.utc()),
|
|
'Central development policy must be closed on API and gateway')
|
|
window = case / 'window.json'
|
|
if case.exists():
|
|
mode = case.lstat()
|
|
require(case.resolve() == case and stat.S_ISDIR(mode.st_mode) and mode.st_uid == 0
|
|
and stat.S_IMODE(mode.st_mode) == 0o700, 'Unsafe development window directory')
|
|
if not window.exists():
|
|
# Even a failed opening can already have registered a real helper journal.
|
|
row = sql(f"""SELECT json_build_object('empty',EXISTS (
|
|
SELECT 1 FROM agent_installations i LEFT JOIN agent_sync_state s USING(tenant_id,agent_id)
|
|
WHERE i.tenant_id='{ids['tenant_id']}' AND i.agent_id='{ids['agent_id']}' AND i.asset_id='{ids['asset_id']}'
|
|
AND i.revoked_at IS NULL AND s.helper_journal_id IS NULL AND coalesce(s.helper_checkpoint,0)=0
|
|
AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.tenant_id=i.tenant_id AND j.agent_id=i.agent_id AND
|
|
(j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown')
|
|
OR convert_from(j.payload,'UTF8')::jsonb->>'execution_identity'='system'))))""")
|
|
require(row['empty'] and action == 'witness', 'A helper or SYSTEM history exists without an opened scope')
|
|
return None
|
|
policy = json.loads(private_read(window))
|
|
require(private_read(case/'closed') == b'closed\n' and private_read(case/'original.json') == current,
|
|
'Expected this closed scope and its original policy')
|
|
only, = policy['scopes']
|
|
hashes = ('helper_binary_sha256','helper_policy_sha256')
|
|
expected_scope = {k:ids[k] for k in ('tenant_id','agent_id','asset_id')}
|
|
expected_scope.update(acceptance_evidence_id=ids['case_id'], principals=[PRINCIPAL], allowed_services=[],
|
|
enrollment_generation=1, assignment_version=1, max_runtime_seconds=30,
|
|
windows_update_max_runtime_seconds=3600)
|
|
for k in hashes:
|
|
require(re.fullmatch('[a-f0-9]{64}',only[k]) and data.get(k,only[k]) == only[k], 'Helper pin differs')
|
|
expected_scope[k] = only[k]
|
|
require(policy['schema_version'] == 3 and policy['profile'] == 'development' and only == expected_scope,
|
|
'Case identity, privileges or original policy differ')
|
|
opened = datetime.fromtimestamp(window.stat().st_mtime, timezone.utc)
|
|
cutoff = min(scope.stamp(policy['expires_at']), datetime.fromtimestamp((case/'closed').stat().st_mtime,timezone.utc))
|
|
require(opened <= cutoff, 'Invalid maintenance horizon')
|
|
query = observation(ids,opened.isoformat(),cutoff.isoformat())
|
|
record = sql(query)
|
|
require(record['open_jobs'] == 0, 'Open jobs require separate reconciliation')
|
|
saved = case/'detach-intent.json'
|
|
done = case/'detached.json'
|
|
receipt = json.loads(private_read(saved)) if saved.exists() else None
|
|
if done.exists():
|
|
require(receipt and json.loads(private_read(done)) == receipt, 'Completion receipt differs from detach intent')
|
|
return receipt
|
|
if record['helper_id'] is None:
|
|
require(receipt and receipt['journal_id'] == record['journal_id'] and record['helper_checkpoint'] == 0,
|
|
'Missing evidence of the original detach transaction')
|
|
if action == 'detach':
|
|
scope.atomic(done,scope.encode(receipt))
|
|
return receipt
|
|
witness = verify(record,ids)
|
|
require(receipt in (None,witness) and data.get('witness',witness) == witness, 'Device or witness changed')
|
|
if action == 'witness':
|
|
return witness
|
|
if receipt is None:
|
|
scope.atomic(saved,scope.encode(witness))
|
|
# Fence a concurrent signed sync (same lock order as the gateway). The server-owned
|
|
# approved jobs, outcomes and signed decisions remain immutable throughout detachment.
|
|
# PostgreSQL jsonb canonical ordering differs from Python: bind our checked snapshot
|
|
# back as JSONB and compare directly under lock, including jobs, grants and head.
|
|
observed_hex = scope.encode(record).hex()
|
|
statements = f"""BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE;
|
|
SET LOCAL lock_timeout='5s'; SET LOCAL statement_timeout='10s';
|
|
DO $$ BEGIN PERFORM 1 FROM agent_installations WHERE tenant_id='{ids['tenant_id']}' AND agent_id='{ids['agent_id']}' FOR UPDATE; END $$;
|
|
WITH checked AS ({query}), changed AS (
|
|
UPDATE agent_sync_state s SET helper_journal_id=NULL,helper_checkpoint=0
|
|
FROM checked WHERE s.tenant_id='{ids['tenant_id']}' AND s.agent_id='{ids['agent_id']}'
|
|
AND s.journal_id='{witness['journal_id']}' AND s.decision_id='{witness['decision_id']}'
|
|
AND s.checkpoint={witness['checkpoint']} AND s.helper_journal_id='{witness['helper_id']}'
|
|
AND s.helper_checkpoint={witness['helper_checkpoint']}
|
|
AND checked.observation = convert_from(decode('{observed_hex}','hex'),'UTF8')::jsonb
|
|
RETURNING 1)
|
|
SELECT to_json(count(*)) FROM changed;
|
|
COMMIT;"""
|
|
require(sql(statements) == 1, 'Concurrent job or sync transition. Inspect the existing intent before retrying')
|
|
scope.atomic(done,scope.encode(witness))
|
|
return witness
|
|
|
|
|
|
def remote_server(action, data):
|
|
"""Run only the two fixed modules over the existing pinned server SSH transport."""
|
|
import dev_stage
|
|
modules = [Path(scope.__file__).read_text(), Path(__file__).read_text()]
|
|
payload = ('import types,sys,json\n'
|
|
'scope=types.ModuleType("windows_update_scope");sys.modules[scope.__name__]=scope\n'
|
|
f'exec({modules[0]!r},scope.__dict__)\n'
|
|
'detach=types.ModuleType("windows_update_detach")\n'
|
|
f'exec({modules[1]!r},detach.__dict__)\n'
|
|
f'print(json.dumps(detach.server({action!r},json.loads({scope.encode(data)!r}))))\n')
|
|
command = shlex.join(['python3','-I','-'])
|
|
return json.loads(subprocess.check_output([*dev_stage.ssh_prefix('server'),command],
|
|
input=payload.encode(),timeout=35))
|