273 lines
15 KiB
Python
273 lines
15 KiB
Python
"""Retry coverage for the HTTPS Linux bootstrap without installing on the test host."""
|
|
from contextlib import ExitStack, redirect_stdout
|
|
from datetime import datetime, timezone
|
|
from importlib.util import module_from_spec, spec_from_file_location
|
|
import io
|
|
import json
|
|
import sqlite3
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import subprocess
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
src = Path(__file__).resolve().parents[1] / 'crates/server/src/bootstrap/lifecycle.py'
|
|
spec = spec_from_file_location('rmm_bootstrap_lifecycle', src)
|
|
lifecycle = module_from_spec(spec)
|
|
spec.loader.exec_module(lifecycle)
|
|
|
|
|
|
class EnrollmentHeartbeat(unittest.TestCase):
|
|
def test_health_probe_waits_for_the_first_certificate_before_fetching(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
state = Path(folder)
|
|
certificate = state / 'certificates/current.json'
|
|
certificate.parent.mkdir()
|
|
now = datetime(2026, 11, 18, 17, tzinfo=timezone.utc)
|
|
record = {'last_contact_at': now.isoformat(),
|
|
'agent_version': '0.1.0-arm64.1', 'connection_generation': 1}
|
|
|
|
def finish_enrollment(_seconds):
|
|
probe.assert_not_called()
|
|
certificate.touch()
|
|
|
|
with patch.object(lifecycle, 'STATE', state), \
|
|
patch.object(lifecycle.time, 'time', return_value=now.timestamp()), \
|
|
patch.object(lifecycle.time, 'monotonic', side_effect=[0, 0, 2]), \
|
|
patch.object(lifecycle.time, 'sleep', side_effect=finish_enrollment) as wait, \
|
|
patch.object(lifecycle, 'installed', return_value=record) as probe, \
|
|
patch.object(lifecycle.subprocess, 'run', return_value=SimpleNamespace(returncode=0)):
|
|
self.assertTrue(lifecycle.healthy('0.1.0-arm64.1', 0))
|
|
wait.assert_called_once_with(2)
|
|
probe.assert_called_once_with()
|
|
|
|
def test_missing_certificate_still_times_out(self):
|
|
with tempfile.TemporaryDirectory() as folder, \
|
|
patch.object(lifecycle, 'STATE', Path(folder)), \
|
|
patch.object(lifecycle.time, 'monotonic', side_effect=[0, 0, 2]), \
|
|
patch.object(lifecycle.time, 'sleep'), \
|
|
patch.object(lifecycle, 'installed', return_value={}) as probe:
|
|
self.assertFalse(lifecycle.healthy('0.1.0-arm64.1', 0, duration=1))
|
|
probe.assert_not_called()
|
|
|
|
|
|
class EnrollmentRetry(unittest.TestCase):
|
|
def setUp(self):
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
folder = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
|
|
self.base = 'https://bootstrap.example'
|
|
self.tenant = '11111111-1111-4111-8111-111111111111'
|
|
root = folder / 'application'
|
|
(root / 'current').mkdir(parents=True)
|
|
(root / 'config.json').write_text(json.dumps({'bootstrap': self.base, 'tenant': self.tenant}))
|
|
(root / 'current/manifest.json').write_text('{"version":"1.0"}')
|
|
(root / 'lifecycle.py').write_bytes(b'previous installer')
|
|
(root / 'release.lock').touch()
|
|
self.state = folder / 'device'
|
|
self.state.mkdir()
|
|
for field, value in [('ROOT',root),('STATE',self.state)]:
|
|
self.stack.enter_context(patch.object(lifecycle,field,value))
|
|
self.enable = self.stack.enter_context(patch.object(lifecycle,'enable'))
|
|
self.provision = self.stack.enter_context(patch.object(lifecycle,'provision_scripts'))
|
|
self.service = self.stack.enter_context(patch.object(lifecycle,'service'))
|
|
self.healthy = self.stack.enter_context(patch.object(lifecycle,'healthy',return_value=True))
|
|
self.save = self.stack.enter_context(patch.object(lifecycle,'save_token'))
|
|
self.stack.enter_context(patch.object(lifecycle.pwd,'getpwnam'))
|
|
self.token = folder / 'retry.token'
|
|
self.token.write_text('0' * 64)
|
|
|
|
def test_unresolved_effects_delay_release_without_blocking_closed_jobs(self):
|
|
with sqlite3.connect(self.state / 'journal.sqlite') as db:
|
|
db.execute('CREATE TABLE job_attempts (state TEXT)')
|
|
for state, expected in [('running','busy'), ('unknown','busy'), ('awaiting_reboot','busy'),
|
|
('closed_unknown','idle'), ('accepted','idle'), ('succeeded','idle')]:
|
|
with self.subTest(state=state):
|
|
db.execute('DELETE FROM job_attempts')
|
|
db.execute('INSERT INTO job_attempts VALUES(?)', (state,))
|
|
db.commit()
|
|
with redirect_stdout(io.StringIO()) as output:
|
|
lifecycle.device_idle()
|
|
self.assertEqual(output.getvalue().strip(), expected)
|
|
|
|
def bootstrap_tenant(self, tenant, expected_platform=None):
|
|
def answer(source, route, target):
|
|
self.assertEqual(source, (self.base,'0' * 64))
|
|
if expected_platform:
|
|
self.assertEqual(route, '/agent/v1/bootstrap/current?platform=' + expected_platform)
|
|
target.write_text(json.dumps({'tenant_id': tenant}))
|
|
return self.stack.enter_context(patch.object(lifecycle,'remote',side_effect=answer))
|
|
|
|
def test_completed_enrollment_keeps_identity_and_needs_no_token(self):
|
|
identity = self.state / 'identity.json'
|
|
identity.write_text('{"agent_id":"original"}')
|
|
remote = self.bootstrap_tenant(self.tenant)
|
|
lifecycle.install(self.base,None)
|
|
self.assertEqual(identity.read_text(), '{"agent_id":"original"}')
|
|
self.assertEqual((lifecycle.ROOT / 'lifecycle.py').read_bytes(), src.read_bytes())
|
|
remote.assert_not_called()
|
|
self.save.assert_not_called()
|
|
self.enable.assert_called_once()
|
|
self.provision.assert_called_once_with(False)
|
|
|
|
def test_expired_initial_token_can_be_replaced_in_same_tenant(self):
|
|
self.bootstrap_tenant(self.tenant)
|
|
lifecycle.install(self.base,self.token)
|
|
self.save.assert_called_once()
|
|
self.healthy.assert_called_once_with('1.0',0)
|
|
|
|
def test_arm64_install_retry_requests_arm64_release(self):
|
|
self.bootstrap_tenant(self.tenant, 'aarch64-unknown-linux-gnu')
|
|
with patch.object(lifecycle.platform, 'machine', return_value='aarch64'):
|
|
lifecycle.install(self.base, self.token, enable_admin_scripts=True)
|
|
self.save.assert_called_once()
|
|
self.provision.assert_called_once_with(True)
|
|
|
|
def test_unsupported_architecture_stops_before_enrollment(self):
|
|
with patch.object(lifecycle.platform, 'machine', return_value='armv7l'), \
|
|
patch.object(lifecycle, 'enrollment_token') as token:
|
|
with self.assertRaisesRegex(ValueError, 'Linux x86-64 or ARM64 required'):
|
|
lifecycle.install(self.base, self.token)
|
|
token.assert_not_called()
|
|
self.enable.assert_not_called()
|
|
|
|
def test_x64_release_is_rejected_before_downloading_to_arm64(self):
|
|
doc = {'manifest': {'release_id': self.tenant, 'platform': 'x86_64-unknown-linux-gnu'}}
|
|
with patch.object(lifecycle.platform, 'machine', return_value='aarch64'), \
|
|
patch.object(lifecycle, 'remote') as download:
|
|
with self.assertRaisesRegex(ValueError, 'release architecture differs'):
|
|
lifecycle.pair(lifecycle.ROOT, doc, (self.base, '0' * 64), lifecycle.ROOT / 'updater')
|
|
download.assert_not_called()
|
|
|
|
def test_timer_starts_only_after_confirmed_first_connection(self):
|
|
self.bootstrap_tenant(self.tenant)
|
|
self.healthy.return_value = False
|
|
with self.assertRaisesRegex(ValueError, 'awaiting authenticated device heartbeat'):
|
|
lifecycle.install(self.base,self.token)
|
|
self.enable.assert_called_once()
|
|
self.service.assert_not_called()
|
|
self.provision.assert_not_called()
|
|
self.healthy.return_value = True
|
|
lifecycle.install(self.base,self.token,enable_admin_scripts=True)
|
|
self.provision.assert_called_once_with(True)
|
|
self.service.assert_called_once_with('enable','--now',lifecycle.TIMER)
|
|
|
|
def test_token_for_another_tenant_cannot_replace_initial_token(self):
|
|
self.bootstrap_tenant('22222222-2222-4222-8222-222222222222')
|
|
with self.assertRaisesRegex(ValueError,'different enrollment tenant'):
|
|
lifecycle.install(self.base,self.token)
|
|
self.save.assert_not_called()
|
|
self.enable.assert_not_called()
|
|
self.assertEqual((lifecycle.ROOT / 'lifecycle.py').read_bytes(), b'previous installer')
|
|
|
|
|
|
class ScriptHelperInstallation(unittest.TestCase):
|
|
def setUp(self):
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
base = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
|
|
self.root = base / 'client'
|
|
self.units = base / 'units'
|
|
self.old = base / 'old-config'
|
|
self.units.mkdir()
|
|
self.uid = SimpleNamespace(pw_uid=990,pw_gid=990)
|
|
self.release = '11111111-1111-4111-8111-111111111111'
|
|
self.previous = '22222222-2222-4222-8222-222222222222'
|
|
self.folder = self.root / 'releases' / self.release
|
|
self.folder.mkdir(parents=True)
|
|
self.manifest = {'release_id': self.release, 'version': 'v2', 'artifacts':[{'role':'helper','sha256':'a'*64}]}
|
|
self.binding = {'tenant_id':self.previous, 'agent_id':self.release, 'public_key_hex':'f'*64}
|
|
self.pin = {'helper_binary_sha256':'a'*64,'policy_sha256':'b'*64}
|
|
lifecycle.save(self.folder / 'manifest.json', self.manifest)
|
|
(self.root / 'release.lock').touch()
|
|
for field, value in [('ROOT',self.root),('UNIT_DIR',self.units),('OLD_HELPER_CONFIG',self.old)]:
|
|
self.stack.enter_context(patch.object(lifecycle,field,value))
|
|
lifecycle.link(self.root, self.release)
|
|
self.unit = self.units / lifecycle.HELPER
|
|
self.service = self.stack.enter_context(patch.object(lifecycle,'service'))
|
|
self.state = self.stack.enter_context(patch.object(lifecycle,'helper_state',return_value='not-found'))
|
|
|
|
def legacy(self, enabled=True):
|
|
self.old.mkdir()
|
|
settings = lifecycle.script_settings(self.binding,self.uid,'c'*64)
|
|
settings['allow_root_scripts'] = enabled
|
|
lifecycle.save(self.old / 'helper.json',settings)
|
|
legacy_image = self.root / 'releases' / self.previous / 'helper'
|
|
self.unit.write_bytes(lifecycle.helper_unit(legacy_image,self.old / 'helper.json'))
|
|
|
|
def test_no_unit_or_explicitly_disabled_local_policy_stays_service_only(self):
|
|
for state, opt_in in [('not-found',False), ('masked',True), ('disabled',True)]:
|
|
with self.subTest(state=state):
|
|
self.state.return_value = state
|
|
lifecycle.provision_scripts(opt_in)
|
|
self.service.assert_not_called()
|
|
self.assertFalse((self.folder / 'script-helper.json').exists())
|
|
|
|
def test_legacy_predecessor_migrates_only_for_exact_authorized_profile(self):
|
|
self.legacy(enabled=False)
|
|
with self.assertRaisesRegex(ValueError,'different binding or profile'):
|
|
lifecycle.helper_unit_files(self.binding,self.uid)
|
|
settings = lifecycle.read(self.old / 'helper.json')
|
|
settings['allow_root_scripts'] = True
|
|
lifecycle.save(self.old / 'helper.json',settings)
|
|
with self.assertRaisesRegex(ValueError,'different binding or profile'):
|
|
lifecycle.helper_unit_files(dict(self.binding, public_key_hex='0'*64),self.uid)
|
|
files = dict(lifecycle.helper_unit_files(self.binding,self.uid))
|
|
self.assertIn(b'lifecycle.py run-helper',files[self.unit])
|
|
self.assertIn(str(self.previous).encode(),self.unit.read_bytes())
|
|
|
|
def test_service_environment_cannot_replace_restrictive_operator_pin_or_claim_a_release(self):
|
|
lifecycle.save(self.root / 'config.json', {'gateway':'https://127.0.0.1','tenant':self.previous})
|
|
lifecycle.save(self.folder / 'script-pin.json',self.pin)
|
|
self.state.return_value = 'enabled'
|
|
environment = {'RMM_FULL_MAINTENANCE_POLICY_FILE':'/etc/site/restricted.json',
|
|
'RMM_SCRIPT_HELPER_RELEASE':self.previous}
|
|
with patch.dict(lifecycle.os.environ,environment,clear=True), patch.object(lifecycle.os,'execv'):
|
|
lifecycle.run_agent()
|
|
self.assertEqual(lifecycle.os.environ['RMM_FULL_MAINTENANCE_POLICY_FILE'],'/etc/site/restricted.json')
|
|
self.assertNotIn('RMM_SCRIPT_HELPER_RELEASE',lifecycle.os.environ)
|
|
with patch.dict(lifecycle.os.environ,{},clear=True), patch.object(lifecycle.os,'execv'):
|
|
lifecycle.run_agent()
|
|
self.assertEqual(lifecycle.os.environ['RMM_SCRIPT_HELPER_RELEASE'],self.release)
|
|
|
|
def test_rejected_pair_does_not_run_candidate_and_altered_policy_stays_intact(self):
|
|
with patch.object(lifecycle.subprocess,'check_output',side_effect=subprocess.CalledProcessError(1,'verify')) as run:
|
|
with self.assertRaises(subprocess.CalledProcessError):
|
|
lifecycle.prepare_script_helper(self.folder,self.binding,self.uid)
|
|
self.assertEqual(run.call_count,1)
|
|
config = self.folder / 'script-helper.json'
|
|
self.assertFalse(config.exists())
|
|
with patch.object(lifecycle.subprocess,'check_output',side_effect=[json.dumps(self.manifest),json.dumps(self.pin).encode()]):
|
|
self.assertEqual(lifecycle.prepare_script_helper(self.folder,self.binding,self.uid),self.pin)
|
|
settings = lifecycle.read(config)
|
|
settings['allow_root_scripts'] = False
|
|
lifecycle.save(config,settings)
|
|
with patch.object(lifecycle.subprocess,'check_output',return_value=json.dumps(self.manifest)) as run:
|
|
with self.assertRaisesRegex(ValueError,'binding changed'):
|
|
lifecycle.prepare_script_helper(self.folder,self.binding,self.uid)
|
|
self.assertEqual(run.call_count,1)
|
|
self.assertEqual(lifecycle.read(config),settings)
|
|
|
|
def test_rollback_selects_previous_pair_and_keeps_pending_until_confirmed(self):
|
|
self.state.return_value = 'enabled'
|
|
previous = self.root / 'releases' / self.previous
|
|
previous.mkdir()
|
|
lifecycle.save(previous / 'manifest.json',dict(self.manifest,release_id=self.previous,version='v1'))
|
|
lifecycle.save(previous / 'script-pin.json',self.pin)
|
|
pending = self.root / 'pending.json'
|
|
intent = {'previous':self.previous,'sequence':2,'script_helper':True}
|
|
lifecycle.save(pending,intent)
|
|
with patch.object(lifecycle,'start_pair') as start, patch.object(lifecycle,'healthy',side_effect=[False,True]):
|
|
with self.assertRaisesRegex(ValueError,'has not yet confirmed'):
|
|
lifecycle.revert(intent)
|
|
self.assertEqual(lifecycle.active_folder(),previous)
|
|
self.assertTrue(pending.exists())
|
|
lifecycle.revert(intent,report_failure=False)
|
|
self.assertFalse(pending.exists())
|
|
self.assertEqual(start.call_count,2)
|
|
self.assertEqual(lifecycle.read(self.root / 'failed.json')['release_sequence'],2)
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|