diff --git a/docs/development.md b/docs/development.md index 1057eb7..0dcd87d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -77,6 +77,14 @@ Auf Linux startet `rmm-discovery-agent.service` als Systemkonto `rmm-discovery` Ein geänderter stabiler Bootstrap-Updater wird ausdrücklich je Mandant rotiert. Dazu beim Veröffentlichen des geprüften neuen Builds `--rotate-bootstrap-from ` ergänzen. Für den Windows-GNU-Worker mit Skript-/Updatesperre verlangt der signierte Release zusätzlich `--minimum-updater-version 2`. Der Befehl lädt den neuen Worker zunächst über die Katalog-API hoch, vergleicht das gespeicherte Objekt und den bisherigen Mandantenpin und schaltet anschließend auf den neuen Hash um. Bereits ausgestellte Installer können den vorher erhaltenen Hash noch 15 Minuten lang abrufen. Weitere Releases desselben Workerstands verwenden nur `--minimum-updater-version 2`. Bestehende Geräte mit älterer stabiler Prüferdatei verwerfen ein Release, dessen signierte Mindestversion ihre eigene Prüferfähigkeit übersteigt. +## Bestehenden verwalteten Linux-Helper auf eine neue Releaseversion umstellen + +Auf der Entwicklungs-VM verweist die Unit `rmm-discovery-managed-script-helper.service` auf den absoluten Pfad der bisherigen signierten Releaseversion. Für den gemeinsamen Versionswechsel vor der Katalogveröffentlichung den aktiven Release aus `/opt/rust-discovery-client/current` erfassen, `rmm-discovery-update.timer` mit systemd stoppen und den Abschluss eines laufenden Update-Dienstes abwarten. Das neue Paar veröffentlichen, den regulären `rmm-discovery-update.service` einmal starten und seine erfolgreich aktivierte Release-ID prüfen. + +Vom Entwicklungsrechner den [Helperwechsel](../tools/dev_linux_helper_rollover.py) zuerst mit `check` und dann mit `switch` aufrufen. Beide Aufrufe erwarten `--tenant UUID --asset UUID --agent UUID --previous ALTE_RELEASE_UUID --current NEUE_RELEASE_UUID`. Der Befehl prüft beide signierten Paare, den vorhandenen Systemdienst, die lokale Gerätebindung und genau einen passenden bestehenden Serverscope. Unter der lokalen Auftrags- und Releasesperre aktualisiert er Helperprofil, Agentpin und Unit-Ziel. Ein neuer authentisierter Kontakt muss beide erwarteten Helperhashes und einen bereiten Abgleich melden. Erst dann ersetzt der Operator im vorhandenen Serverscope die beiden Hashwerte. Principals, erlaubte Aktionen, übrige Geräte und Kunden bleiben erhalten. + +Nach erfolgreichem Rücklesen den Update-Timer wieder starten. Ein unterbrochener `switch` ist mit denselben Kennungen fortsetzbar. `rollback` stellt unter denselben Prüfungen den vorherigen Helper und dessen Pins wieder her und belässt den Agenten in der angegebenen aktuellen Releaseversion. Bei ausstehendem Abgleich bleibt der bestehende Serverscope wirksam. Der stabile Verifier unter `/opt/rust-discovery-client/updater` wird separat anhand seines Bootstrap-Pins ausgetauscht. + ## Stabilen Windows-Updater im Entwicklungsbetrieb ersetzen Für den gezielten Austausch des stabilen Workers zuerst die fertige Datei mit `tools/dev_stage.py windows DATEI` übertragen. [Replace-DevelopmentReleaseUpdater.ps1](../deploy/windows/Replace-DevelopmentReleaseUpdater.ps1) erwartet `-Candidate`, die beiden exakten Hashes über `-ExpectedPreviousSha256` und `-ExpectedCandidateSha256`, `-AgentId` und die zu prüfenden `-VerifyReleaseIds`. In einer administrativen Windows-PowerShell ausführen. Der Befehl verlangt einen laufenden Agenten und gestoppten Updater, prüft die signierten Releasepaare nativ, ersetzt den Worker atomar mit Sicherung und startet den Updatedienst mit SCM-Wiederanlauf. Bei standardmäßig Restricted geltender Skriptrichtlinie für die Entwicklungs-PowerShell `Set-ExecutionPolicy -Scope Process -ExecutionPolicy RemoteSigned` verwenden. diff --git a/tests/linux_managed_helper_rollover.py b/tests/linux_managed_helper_rollover.py new file mode 100644 index 0000000..8abf1c9 --- /dev/null +++ b/tests/linux_managed_helper_rollover.py @@ -0,0 +1,136 @@ +"""Restart and access-boundary regressions for the managed development helper.""" +import copy +import hashlib +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace +import tempfile +import unittest +from contextlib import ExitStack +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('rollover', Path(__file__).resolve().parents[1] / + 'tools/dev_linux_helper_rollover.py') +op = importlib.util.module_from_spec(spec) +spec.loader.exec_module(op) + + +class LinuxHelperSwitch(unittest.TestCase): + def setUp(self): + stack = ExitStack() + self.addCleanup(stack.close) + root = Path(stack.enter_context(tempfile.TemporaryDirectory())) + cfg = root/'etc' + cfg.mkdir() + self.unit = root/'helper.service' + self.proof, self.settings = cfg/'agent-pin.json', cfg/'helper.json' + self.args = SimpleNamespace(tenant='tenant', asset='asset', agent='agent', + previous='old', current='new', action='switch') + self.old, self.new = root/'old', root/'new' + self.old.mkdir() + self.new.mkdir() + (root/'release.lock').touch() + (root/'config.json').write_bytes(op.encoded({'tenant': 'tenant'})) + (root/'current').symlink_to(self.new) + self.binding = dict(tenant_id='tenant', asset_id='asset', agent_id='agent', + enrollment_generation=1, assignment_version=1) + self.hashes = ['1'*64, '2'*64] + locked = dict(binding=self.binding, profile='full_maintenance_v1', allowed_services=[], + allow_reboot=False, allow_root_scripts=True, helper_binary_sha256=self.hashes[0]) + self.settings.write_bytes(op.encoded(locked)) + self.proof.write_bytes(self.sign(locked)) + self.unit.write_text(f'[Service]\nExecStart={self.old}/helper\nOther=unchanged\n') + self.live = {self.unit.name:self.old/'helper', op.AGENT:self.new/'agent'} + self.restart_count = 0 + for name, value in [('ROOT',root),('CONFIG',cfg),('UNIT',self.unit), + ('protected',lambda path: None),('process_image',self.live.get), + ('run',self.command),('prop',self.prop),('pair',self.pair)]: + stack.enter_context(patch.object(op, name, value)) + stack.enter_context(patch.object(op.os,'geteuid',return_value=0)) + self.before = [p.read_bytes() for p in [self.settings,self.proof,self.unit]] + + def sign(self, config): + sha = hashlib.sha256(op.encoded(config)).hexdigest() + return op.encoded(dict(identity=self.binding,policy_sha256=sha, + helper_binary_sha256=config['helper_binary_sha256'])) + + def pair(self, release): + idx = int(release == 'new') + return (self.old, self.new)[idx], {'release_sequence':1+idx,'version':str(idx)}, self.hashes[idx] + + def prop(self, unit, field): + if field == 'ActiveState': + return 'inactive' + if field == 'FragmentPath': + return str(self.unit) + if field == 'ExecStart': + path = self.unit.read_text().split('ExecStart=')[1].splitlines()[0] + return f'{{ path={path} ; argv[]={path} ; }}' + if field == 'Environment': + return 'RMM_HELPER_CONFIG='+str(self.settings) + raise AssertionError(field) + + def command(self,*args,**kw): + if args[0] == 'runuser': + if args[-1] == 'idle': + return b'idle' + return op.encoded({'asset_id':'asset','agent_id':'agent'}) + if args[1] == '--inspect-full-maintenance-profile': + return self.sign(json.loads(Path(kw['env']['RMM_HELPER_CONFIG']).read_bytes())) + if args == ('systemctl','stop',op.AGENT,self.unit.name): + self.live.clear() + elif args == ('systemctl','start',self.unit.name,op.AGENT): + self.restart_count += 1 + path = self.unit.read_text().split('ExecStart=')[1].splitlines()[0] + self.live.update({self.unit.name:Path(path),op.AGENT:self.new/'agent'}) + elif args != ('systemctl','daemon-reload'): + raise AssertionError(args) + return b'' + + def test_interrupted_switch_resume_and_exact_rollback(self): + report = op.device(self.args,False) + self.assertEqual(self.before,[p.read_bytes() for p in [self.settings,self.proof,self.unit]]) + original = op.replace + def interrupt(path,data): + if path == self.proof: + raise RuntimeError('interrupted') + original(path,data) + with patch.object(op,'replace',interrupt), self.assertRaisesRegex(RuntimeError,'interrupted'): + op.device(self.args,True) + self.assertEqual(report,op.device(self.args,True)) + self.assertEqual(report,op.device(self.args,True)) + self.assertEqual(1,self.restart_count) + self.args.action='rollback' + op.device(self.args,True) + self.assertEqual(self.before,[p.read_bytes() for p in [self.settings,self.proof,self.unit]]) + self.assertEqual(self.live[op.AGENT],self.new/'agent') + + def test_mismatched_proof_or_unit_rejected_before_service_changes(self): + self.proof.write_text('{}') + with self.assertRaisesRegex(ValueError,'proof'): + op.device(self.args,True) + self.proof.write_bytes(self.before[1]) + self.unit.write_text('ExecStart=/unrelated/helper\n') + with self.assertRaisesRegex(ValueError,'effective'): + op.device(self.args,True) + self.assertEqual(self.restart_count,0) + + def test_only_two_existing_scope_pins_change(self): + receipt=op.device(self.args,False) + scope={**self.binding, 'principals':['existing-operator'], 'allow_reboot':False, + **dict(zip(op.PINS,receipt['previous']))} + policy={'schema_version':1,'linux':[scope],'windows':[{'agent_id':'other'}]} + before=copy.deepcopy(policy) + op.existing_scope(policy,receipt).update(zip(op.PINS,receipt['current'])) + changed={k for k in scope if scope[k]!=before['linux'][0][k]} + self.assertEqual(changed,set(op.PINS)) + self.assertEqual(policy['windows'],before['windows']) + for bad in [dict(receipt,binding=dict(receipt['binding'],asset_id='other')), + dict(receipt,current=['foreign','foreign'])]: + with self.assertRaises(ValueError): + op.existing_scope(policy,bad) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/dev_linux_helper_rollover.py b/tools/dev_linux_helper_rollover.py new file mode 100644 index 0000000..25b082e --- /dev/null +++ b/tools/dev_linux_helper_rollover.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Rotate the existing Linux development helper after the signed agent pair has switched.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import re +import shlex +import stat +import subprocess +import sys +import tempfile +import time +import uuid + + +ROOT = Path('/opt/rust-discovery-client') +CONFIG = Path('/etc/rmm-discovery-managed-scripts') +UNIT = Path('/etc/systemd/system/rmm-discovery-managed-script-helper.service') +AGENT = 'rmm-discovery-agent.service' +TIMER = 'rmm-discovery-update.timer' +POLICY = Path('/opt/rust-discovery/devstate/managed-scripts/policy.json') +HASH = re.compile(r'[0-9a-f]{64}') +PINS = ('helper_binary_sha256', 'helper_policy_sha256') + + +def require(test, error): + if not test: + raise ValueError(error) + + +def encoded(obj): + return (json.dumps(obj, separators=(',', ':')) + '\n').encode() + + +def protected(path, uid=0): + for parent in path.parents: + info = parent.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid in (0, uid) and not info.st_mode & 0o022, + f'Untrusted directory: {parent}') + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == uid and not info.st_mode & 0o022, + f'Untrusted file: {path}') + return info + + +def replace(path, data): + info = path.lstat() + if path.read_bytes() == data: + return + fd, name = tempfile.mkstemp(dir=path.parent) + try: + with os.fdopen(fd, 'wb') as stream: + os.fchown(stream.fileno(), info.st_uid, info.st_gid) + os.fchmod(stream.fileno(), stat.S_IMODE(info.st_mode)) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(name, path) + finally: + Path(name).unlink(missing_ok=True) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +def run(*args, **kwargs): + return subprocess.check_output(args, timeout=120, **kwargs) + + +def prop(unit, field): + return run('systemctl', 'show', '--property', field, '--value', unit).decode().strip() + + +def process_image(service): + pid = int(prop(service, 'MainPID')) + return Path(f'/proc/{pid}/exe').resolve(strict=True) if pid else None + + +def pair(release_id): + release = ROOT / 'releases' / release_id + names = ('envelope.json', 'agent', 'helper', 'manifest.json') + for name in names: + protected(release / name) + manifest = json.loads(run(ROOT / 'updater', '--verify-bootstrap', ROOT / 'keyring.json', + release / 'envelope.json', release / 'agent', release / 'helper')) + require(manifest == json.loads((release / 'manifest.json').read_bytes()) + and manifest['release_id'] == release_id and manifest['platform'] == 'x86_64-unknown-linux-gnu', + 'Signed release binding mismatch') + helper, = [x for x in manifest['artifacts'] if x['role'] == 'helper'] + return release, manifest, helper['sha256'] + + +def repoint_unit(contents, old, new, destination): + lines = contents.splitlines(keepends=True) + paths = [f'ExecStart={path}/helper\n'.encode() for path in (old, new)] + starts = [i for i, line in enumerate(lines) if line.startswith(b'ExecStart=')] + require(len(starts) == 1 and lines[starts[0]] in paths, 'Unexpected managed helper ExecStart') + lines[starts[0]] = f'ExecStart={destination}/helper\n'.encode() + return b''.join(lines) + + +def device(args, activate): + require(os.geteuid() == 0, 'Linux maintenance requires the development administrator') + for path in [ROOT / n for n in ('updater', 'keyring.json', 'release.lock', 'config.json', 'lifecycle.py')]: + protected(path) + with (ROOT / 'release.lock').open('rb') as guard: + fcntl.flock(guard, fcntl.LOCK_EX | fcntl.LOCK_NB) + require(prop(TIMER, 'ActiveState') == 'inactive' and + prop('rmm-discovery-update.service', 'ActiveState') == 'inactive', + 'Pause the update timer and wait for the running release check') + old, old_manifest, old_hash = pair(args.previous) + new, manifest, new_hash = pair(args.current) + require((ROOT/'current').resolve(strict=True) == new and old_hash != new_hash and + manifest['release_sequence'] > old_manifest['release_sequence'], 'Unexpected active release pair') + require(json.loads((ROOT/'config.json').read_bytes())['tenant'] == args.tenant, 'Wrong installed tenant') + identity = json.loads(run('runuser', '-u', 'rmm-discovery', '--', 'cat', + '/var/lib/rmm-discovery/identity.json')) + require(identity['agent_id'] == args.agent and identity['asset_id'] == args.asset, 'Wrong device') + require(run('runuser', '-u', 'rmm-discovery', '--', 'python3', '-I', ROOT/'lifecycle.py', 'idle').strip() + == b'idle', 'Unresolved device attempt') + cfg, proof = CONFIG/'helper.json', CONFIG/'agent-pin.json' + for path in (UNIT, cfg, proof): + protected(path) + command = prop(UNIT.name, 'ExecStart') + paths = (old/'helper', new/'helper') + effective = [path for path in paths if f'path={path} ; argv[]={path} ;' in command] + require(prop(UNIT.name, 'FragmentPath') == str(UNIT) and len(effective) == 1 and + command.count('argv[]=') == 1 and + 'RMM_HELPER_CONFIG=' + str(cfg) in shlex.split(prop(UNIT.name, 'Environment')), + 'Unexpected effective managed helper configuration') + require(process_image(UNIT.name) in (*paths, None) and process_image(AGENT) in (new/'agent', None), + 'Running process does not match either signed generation') + settings = json.loads(cfg.read_bytes()) + binding = {k: settings['binding'][k] for k in + ('tenant_id', 'agent_id', 'asset_id', 'enrollment_generation', 'assignment_version')} + require((binding['tenant_id'], binding['asset_id'], binding['agent_id']) == (args.tenant, args.asset, args.agent) + and settings['profile'] == 'full_maintenance_v1' and settings[PINS[0]] in (old_hash, new_hash), + 'Wrong managed profile or helper hash') + pin = json.loads(proof.read_bytes()) + candidates = [] + with tempfile.TemporaryDirectory(dir=CONFIG) as work: + temp = Path(work)/'helper.json' + for folder, binary in [(old, old_hash), (new, new_hash)]: + profile = encoded(dict(settings, helper_binary_sha256=binary)) + temp.write_bytes(profile) + temp.chmod(0o600) + native = run(folder/'helper', '--inspect-full-maintenance-profile', + env={'PATH':'/usr/bin:/bin', 'RMM_HELPER_CONFIG':str(temp)}) + computed = json.loads(native) + require(computed['helper_binary_sha256'] == binary and HASH.fullmatch(computed['policy_sha256']), + 'Native profile verification failed') + candidates.append((profile, native, computed)) + source, target = candidates + require(pin in [source[2], target[2]] and + {k:v for k,v in source[2].items() if k not in ('policy_sha256', PINS[0])} == + {k:v for k,v in target[2].items() if k not in ('policy_sha256', PINS[0])}, + 'Existing and proposed proof must bind the same profile and identity') + undo = args.action == 'rollback' + profile, native, chosen = source if undo else target + destination = old if undo else new + unit = repoint_unit(UNIT.read_bytes(), old, new, destination) + if activate: + running = process_image(UNIT.name) == destination/'helper' and process_image(AGENT) == new/'agent' + if not (settings[PINS[0]] == chosen[PINS[0]] and pin == chosen and UNIT.read_bytes() == unit and running): + run('systemctl', 'stop', AGENT, UNIT.name) + replace(cfg, profile) + replace(proof, native) + replace(UNIT, unit) + run('systemctl', 'daemon-reload') + run('systemctl', 'start', UNIT.name, AGENT) + for service, expected in [(UNIT.name, destination/'helper'), (AGENT, new/'agent')]: + require(process_image(service) == expected, 'Unexpected running binary') + return dict(binding=binding, version=manifest['version'], previous=[old_hash, source[2]['policy_sha256']], + current=[new_hash, target[2]['policy_sha256']]) + + +def existing_scope(policy, receipt): + require(policy['schema_version'] == 1, 'Unknown scope format') + scopes = [s for s in policy['linux'] if s['agent_id'] == receipt['binding']['agent_id']] + require(len(scopes) == 1, 'Exactly one pre-existing scope is required') + scope = scopes[0] + require(all(scope[k] == v for k,v in receipt['binding'].items()) and + [scope[k] for k in PINS] in (receipt['previous'], receipt['current']), 'Existing scope changed') + return scope + + +def server(args, receipt, activate): + require([receipt['binding'][k] for k in ('tenant_id','asset_id','agent_id')] == + [args.tenant,args.asset,args.agent], 'Wrong scope receipt') + for pins in [receipt['previous'], receipt['current']]: + require(len(pins) == 2 and all(HASH.fullmatch(x) for x in pins), 'Invalid helper hashes') + owner = (POLICY.parent.parent/'jobs/ed25519.pem').stat().st_uid + directory = POLICY.parent.lstat() + require(stat.S_ISDIR(directory.st_mode) and directory.st_uid == owner and not directory.st_mode & 0o077, + 'Invalid managed policy directory') + lock = POLICY.parent/'operator.lock' + for path in (POLICY, lock): + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid in (0, owner) and not info.st_mode & 0o022, + 'Invalid managed policy file') + wanted = receipt['previous' if args.action == 'rollback' else 'current'] + query = """BEGIN READ ONLY; +SELECT json_build_object('generation',enrollment_generation,'assignment',assignment_version, + 'fresh',sync_ready AND helper_sync_ready AND NOT restore_blocked + AND last_authenticated_at > now()-interval '2 minutes' AND certificate_sha256=last_authenticated_certificate_sha256, + 'version',reported_agent_version,'caps',capabilities, + 'idle',NOT EXISTS(SELECT 1 FROM jobs j WHERE j.agent_id=i.agent_id AND j.tenant_id=i.tenant_id AND + j.state NOT IN ('succeeded','failed','cancelled','expired','closed_unknown'))) +FROM agent_installations i WHERE tenant_id=:'tenant' AND asset_id=:'asset' AND agent_id=:'agent' + AND revoked_at IS NULL; +COMMIT;""" + deadline = time.monotonic()+120 + while True: + with lock.open('rb') as guard: + fcntl.flock(guard, fcntl.LOCK_EX) + policy = json.loads(POLICY.read_bytes()) + scope = existing_scope(policy, receipt) + state = json.loads(run('docker','exec','-i','rust-discovery-server-postgres-1','psql','-XqAt', + '-U','postgres','-d','rmm','-v','ON_ERROR_STOP=1', + '-v',f'tenant={args.tenant}','-v',f'asset={args.asset}','-v',f'agent={args.agent}', + input=query.encode())) + require(state['idle'] and state['generation'] == scope['enrollment_generation'] and + state['assignment'] == scope['assignment_version'], 'Unresolved job or changed enrollment') + if not activate: + return {'existing_scope':True} + if state['fresh'] and state['version'] == receipt['version'] and all( + p in state['caps'] for p in ['helper.binary.sha256:'+wanted[0],'helper.policy.sha256:'+wanted[1]]): + if [scope[k] for k in PINS] != wanted: + scope.update(zip(PINS,wanted)) + replace(POLICY, encoded(policy)) + return {'scoped_helper_binary':wanted[0],'scoped_helper_policy':wanted[1]} + require(time.monotonic()