#!/usr/bin/env python3 """One frozen Script1.20/MSI1.19 server upgrade; no jobs, approvals or policies. prepare is offline. inspect/status are remote read-only. build/deploy each launch one durable, create-once case worker. Lost replies permit status only, not retry. The notebook never runs an RMM server/client. Existing deployment tools remain unchanged. See docs/script-console-cloud-upgrade-v1.md before actual use. """ import argparse import base64 import hashlib import inspect import io import json import os from pathlib import Path import re import tarfile import uuid import remote_client_bootstrap as bootstrap ROOT = Path(__file__).resolve().parents[1] STATE = ROOT / 'devstate/script-console-cloud-upgrade-v1' FROZEN = ROOT / 'devstate/script-console-qualification/run-29461892-343f-42f4-b469-88703535425e/manifest.json' FROZEN_SHA = 'ddd1a78d1cd289b02b1f98572f9652171ec00be84e01ce099fd722d689bcaacd' OLD_IMAGE = 'sha256:25697f948325a11f755893e16ec5be2da138c3333c8e60f75a18f973d6ac26e2' EXTRA = { 'deploy/Dockerfile': 'd41b4ec902c11a9e25c22b1c1d3d60d15afc207f2598ecf559cf3bdae4f8e782', '.dockerignore': 'fdb5cba5967249f5dac35d45c5edf0be2a4e1594c94d04a2ed416d89821ace75', 'deploy/server.compose.yaml': 'd2044b0ee6d584f360990635bec149511d5a0623ec562976541900d751998818', } WEB = { 'index.html': '95d6f9058cc713c6af36fbd22ccc60678f081765a853b53d19ac6f0d219804f8', 'assets/index-Be1zm1OZ.css': '916e8870d4e9a9553de452b3ff6c71671bb6a7a22355de0aff432db516a9f3e7', 'assets/index-CTgvl2jr.js': 'f611662042eb4cc44f25c7d5ddc0d2d0562e32be267d3d5fb1db31b0d310f879', } SUPPORT = { 'deploy/remote_client_bootstrap.py': '785c43a51dfe3f93e39798dbd7611c353752ae2fc64b22e41ba168e221620d30', 'deploy/vm_access_check.py': '5c8107ea9151a9f498b2530cdc61c1a7a6d745aff853a9b05710f3395838c0dd', } require = bootstrap.require def sha(raw): return hashlib.sha256(raw).hexdigest() def encoded(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() def create(path, raw): fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, 'wb') as stream: stream.write(raw) stream.flush() os.fsync(stream.fileno()) fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) try: os.fsync(fd) finally: os.close(fd) def remote(payload): """Self-contained function transported by the existing strict-hostkey RPC.""" import datetime as dt import fcntl import http.client import io import platform import socket import ssl import stat import subprocess import tarfile import time import traceback os.umask(0o077) root = Path('/opt/rust-discovery') action = payload['action'] require(os.geteuid() == 0 and platform.machine() == 'aarch64', 'fixed root ARM64 server') require(action in ('inspect', 'build', 'build-worker', 'deploy', 'deploy-worker', 'status'), 'fixed upgrade action') ident = str(uuid.UUID(payload['id'])) require(ident == payload['id'], 'canonical case UUID') folder = root / 'devstate' / ('script-console-upgrade-v1-' + ident) compose = ['docker', 'compose', '--env-file', str(root / 'devstate/.env'), '-f', str(root / 'deploy/server.compose.yaml')] def digest(raw): return hashlib.sha256(raw).hexdigest() def enc(value): return json.dumps(value, sort_keys=True, separators=(',', ':')).encode() def run(args, timeout=20, **kwargs): result = subprocess.run(args, capture_output=True, timeout=timeout, **kwargs) require(result.returncode == 0, 'fixed server command incomplete; inspect private case') return result.stdout def owned(path, directory=False): value = path.lstat() require(value.st_uid == 0 and not value.st_mode & 0o022 and not path.is_symlink() and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'root-owned fixed path') return value def raw(path, limit=2 * 1024 * 1024): value = owned(path) require(value.st_size <= limit and value.st_nlink == 1, 'bounded private regular file') return path.read_bytes() def syncdir(path): fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) try: os.fsync(fd) finally: os.close(fd) def once(path, data, mode=0o600): fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode) with os.fdopen(fd, 'wb') as stream: stream.write(data) stream.flush() os.fchmod(stream.fileno(), mode) os.fsync(stream.fileno()) syncdir(path.parent) def load(path): return json.loads(raw(path, 8 * 1024 * 1024)) def image(tag): # Separate fields avoid ever retrieving environment variables/secrets. value = run(['docker', 'image', 'inspect', tag, '--format', '{{.Id}} {{.Architecture}} {{.Os}}']).decode().split() require(len(value) == 3 and value[1:] == ['arm64', 'linux'] and value[0].startswith('sha256:'), 'native ARM64 image') return value[0] def closed_root_policy(): # Existing MVP state: flag=true, but this exact historical policy is # expired. Neither the flag nor the policy is changed by deployment. directory = root / 'devstate/root-policy' path = directory / 'policy.json' parent = directory.lstat(); value = path.lstat() require(not directory.is_symlink() and stat.S_ISDIR(parent.st_mode) and parent.st_uid == 10001 and not parent.st_mode & 0o022 and not path.is_symlink() and stat.S_ISREG(value.st_mode) and value.st_uid == 10001 and stat.S_IMODE(value.st_mode) == 0o600 and value.st_nlink == 1 and 0 < value.st_size <= 65536, 'fixed protected closed Root policy path') data = path.read_bytes() expected = 'cf15cbb68b590475df6def28e7dcffb7d88e0ca2d2555a06174fb654eeff05f4' require(digest(data) == expected, 'exact historical closed Root policy hash') policy = json.loads(data) require(isinstance(policy, dict) and set(policy) == {'schema_version', 'profile', 'expires_at', 'scopes'} and type(policy['schema_version']) is int and policy['schema_version'] == 1 and policy['profile'] == 'development' and isinstance(policy['expires_at'], str) and isinstance(policy['scopes'], list) and 1 <= len(policy['scopes']) <= 64 and all(isinstance(scope, dict) for scope in policy['scopes']), 'exact legacy Root policy schema') expiry = dt.datetime.fromisoformat(policy['expires_at']) require(expiry.utcoffset() is not None and expiry <= dt.datetime.now(dt.timezone.utc), 'expired Root policy denies every scope') return {'sha256': expected, 'schema_version': 1, 'profile': 'development', 'expires_at': policy['expires_at'], 'closed': True} def container(service): values = run(compose + ['ps', '-a', '-q', service]).decode().split() require(len(values) == 1, 'one fixed compose service') cid = values[0] out = {'id': cid} for name, fmt in (('image', '.Image'), ('user', '.Config.User'), ('running', '.State.Running'), ('mounts', '.Mounts'), ('ports', '.HostConfig.PortBindings')): out[name] = json.loads(run(['docker', 'inspect', cid, '--format', '{{json ' + fmt + '}}'])) if service in ('api', 'gateway'): # Filter inside Docker's template: credential env values never # enter Python/stdout. No .env, token or private key is opened. names = ('RMM_PROFILE', 'RMM_DEV_AUTH', 'RMM_SERVER_ROLE', 'RMM_UI_ORIGIN', 'RMM_DEV_ROOT_ENABLED', 'RMM_DEV_SYSTEM_ENABLED', 'RMM_DEV_DELIVERY_FAULTS', 'RMM_DEV_DELIVERY_FAULTS_FILE', 'RMM_DEV_SYSTEM_POLICY_FILE', 'RMM_DEV_ROOT_POLICY_FILE') select = ' '.join('(eq $key "' + name + '")' for name in names) fmt = '{{range .Config.Env}}{{$key := index (split . "=") 0}}{{if or ' + select + '}}{{println .}}{{end}}{{end}}' env = dict(line.split('=', 1) for line in run(['docker', 'inspect', cid, '--format', fmt]).decode().splitlines() if '=' in line) out['settings'] = {key: env.get(key) for key in names} require(env.get('RMM_PROFILE') == 'development' and env.get('RMM_DEV_AUTH') == 'true' and env.get('RMM_SERVER_ROLE') == service and env.get('RMM_UI_ORIGIN') == 'https://localhost:18443' and env.get('RMM_DEV_ROOT_ENABLED') == 'true' and env.get('RMM_DEV_ROOT_POLICY_FILE') == '/root-policy/policy.json' and env.get('RMM_DEV_SYSTEM_ENABLED') is None and env.get('RMM_DEV_DELIVERY_FAULTS') == 'false' and env.get('RMM_DEV_DELIVERY_FAULTS_FILE') is None and env.get('RMM_DEV_SYSTEM_POLICY_FILE') is None, 'unchanged private dev/test role with Root/SYSTEM/fault gates closed') mounts = [mount for mount in out['mounts'] if mount['Destination'] == '/root-policy'] require(len(mounts) == 1 and mounts[0]['Type'] == 'bind' and mounts[0]['RW'] is False and mounts[0]['Source'] == str(root / 'devstate/root-policy'), 'exact read-only Root policy mount') out['closed_root_policy_sha256'] = closed_root_policy()['sha256'] require(out['user'] == '10001:10001', 'unprivileged application role') port = '18443' if service == 'api' else '18444' require(out['ports'] == {'18443/tcp': [{'HostIp': '127.0.0.1', 'HostPort': port}]}, 'loopback-only role') else: require(not out['ports'], 'PostgreSQL has no published port') return out def controls(): names = ('deploy/Dockerfile', '.dockerignore', 'deploy/server.compose.yaml') files = {name: digest(raw(root / name)) for name in names} for name in ('rmm-server-ingress.socket', 'rmm-server-ingress.service'): files['/etc/systemd/system/' + name] = digest(raw(Path('/etc/systemd/system') / name)) # Metadata-only identity of secret trees. Private material is never read. identities = {} for name in ('devstate/.env', 'devstate/pki', 'devstate/api-pki', 'devstate/jobs', 'devstate/root-policy'): path = root / name paths = [path, *sorted(path.rglob('*'))] if path.is_dir() else [path] for item in paths: value = item.lstat() require(not item.is_symlink() and (stat.S_ISREG(value.st_mode) or stat.S_ISDIR(value.st_mode)), 'preserved state object') identities[str(item.relative_to(root))] = [value.st_dev, value.st_ino, value.st_uid, value.st_gid, value.st_mode, value.st_size if stat.S_ISREG(value.st_mode) else None, value.st_mtime_ns if stat.S_ISREG(value.st_mode) else None] ingress = run(['systemctl', 'show', 'rmm-server-ingress.socket', '-p', 'ActiveState', '-p', 'UnitFileState']).decode() require('ActiveState=active' in ingress, 'existing public socket ingress stays active') return {'files': files, 'protected_metadata': identities, 'ingress': ingress, 'root_policy': closed_root_policy()} def sql(query): return json.loads(run(compose + ['exec', '-T', 'postgres', 'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm'], timeout=30, input=query.encode())) def database(): # One repeatable-read snapshot, only counts and SHA256 values leave PG. tables = ('tenants', 'assets', 'operation_plans', 'approval_requests', 'authorizations', 'operations', 'jobs', 'outbox', 'execution_permits', 'job_receipts', 'audit_events', 'operation_submissions', 'live_query_requests', 'memberships', 'tenant_limits', 'oidc_principals', 'package_recipes', 'artifacts') pairs = [] for table in tables: pairs.append("'" + table + "',(SELECT json_build_object('count',count(*),'sha256',encode(sha256(convert_to(" "coalesce(string_agg(to_jsonb(t)::text,E'\\n' ORDER BY to_jsonb(t)::text),''),'UTF8')),'hex')) FROM " + table + ' t)') # Stable device identity excludes expected changing passive Sync/contact. identity = "(SELECT coalesce(json_agg(q ORDER BY tenant_id,agent_id),'[]'::json) FROM (SELECT tenant_id,agent_id,asset_id,enrollment_generation,assignment_version,certificate_sha256,revoked_at FROM agent_installations) q)" migrations = "(SELECT json_agg(q ORDER BY version) FROM (SELECT version,description,installed_on,success,encode(checksum,'hex') AS checksum,execution_time FROM _sqlx_migrations) q)" role = "(SELECT json_build_object('superuser',rolsuper,'bypassrls',rolbypassrls) FROM pg_roles WHERE rolname='rmm_app')" query = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SELECT json_build_object('history',json_build_object(" + ','.join(pairs) + "),'identities'," + identity + ",'migrations'," + migrations + ",'app_role'," + role + ",'unresolved',(SELECT count(*) FROM jobs WHERE state NOT IN ('succeeded','failed','cancelled','expired'))); COMMIT;" out = sql(query) require(out['unresolved'] == 0 and out['app_role'] == {'superuser': False, 'bypassrls': False}, 'settled history and no RLS bypass') require(all(row['success'] for row in out['migrations']), 'successful SQLx ledger') return out def witness(running=True): current = {'runtime_image': image('rust-discovery-server:local'), 'controls': controls(), 'containers': {name: container(name) for name in ('postgres', 'api', 'gateway')}, 'index_sha256': digest(raw(root / 'web/dist/index.html')), 'database': database()} pg = current['containers']['postgres'] mounts = [m for m in pg['mounts'] if m['Destination'] == '/var/lib/postgresql/data'] require(len(mounts) == 1 and mounts[0]['Type'] == 'volume' and pg['running'], 'existing PG data volume') value = Path(mounts[0]['Source']).lstat() require(stat.S_ISDIR(value.st_mode) and not Path(mounts[0]['Source']).is_symlink(), 'same PG data directory') current['pg_volume'] = {'mount': mounts[0], 'device': value.st_dev, 'inode': value.st_ino} for name in ('api', 'gateway'): require(current['containers'][name]['running'] == running, 'fixed application running state') require(current['containers'][name]['image'] == current['runtime_image'], 'both services run runtime tag') return current def before_guard(now, expected, stopped=False): require(now['runtime_image'] == expected['runtime_image'] == payload['old_image'], 'runtime image CAS') for key in ('controls', 'index_sha256', 'database', 'pg_volume'): require(now[key] == expected[key], 'preserved ' + key + ' CAS') for name in ('postgres', 'api', 'gateway'): current = dict(now['containers'][name]) old = dict(expected['containers'][name]) if stopped and name != 'postgres': current.pop('running'); old.pop('running') require(current == old, 'fixed prior service/container CAS') def migration_guard(record, after=False): expected = record['migration_checksums'] require([row['version'] for row in record['baseline']['database']['migrations']] == list(range(1, 14)), 'exact prior migrations 1..13') for row in record['baseline']['database']['migrations']: require(row['checksum'] == expected[str(row['version'])], 'frozen old migration checksum') if after: now = database() require(now['migrations'][:13] == record['baseline']['database']['migrations'] and [row['version'] for row in now['migrations']] == list(range(1, 15)) and now['migrations'][13]['checksum'] == expected['14'], 'only additive frozen migration14') for key in ('history', 'identities', 'app_role', 'unresolved'): require(now[key] == record['baseline']['database'][key], 'original database rows unchanged') check = sql("BEGIN READ ONLY; SELECT json_build_object('count',(SELECT count(*) FROM windows_msi_recipes)," "'rls',(SELECT relrowsecurity AND relforcerowsecurity FROM pg_class WHERE oid='windows_msi_recipes'::regclass)," "'select',has_table_privilege('rmm_app','windows_msi_recipes','SELECT')," "'insert',has_table_privilege('rmm_app','windows_msi_recipes','INSERT')," "'update',has_table_privilege('rmm_app','windows_msi_recipes','UPDATE')," "'delete',has_table_privilege('rmm_app','windows_msi_recipes','DELETE')," "'trigger',EXISTS(SELECT 1 FROM pg_trigger WHERE tgrelid='windows_msi_recipes'::regclass AND tgname='immutable_content' AND tgenabled='O')); COMMIT;") require(check == {'count': 0, 'rls': True, 'select': True, 'insert': True, 'update': False, 'delete': False, 'trigger': True}, 'empty immutable MSI tenant catalog with FORCE RLS') return now def check_source(record): require(all(digest(raw(folder / 'context' / name)) == value for name, value in record['source_files'].items()), 'frozen build context unchanged') def https(port, path, host='localhost'): require((port, host) in ((18443, 'localhost'), (18444, 'localhost'), (443, 'discovery.ease-technologies.de')), 'fixed role TLS endpoint and name') ca = root / ('devstate/api-pki/ca.pem' if port == 18443 else 'devstate/pki/ca.pem') class HTTPS(http.client.HTTPSConnection): def connect(self): self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', port), timeout=5), server_hostname=self.host) client = HTTPS(host, port, timeout=5, context=ssl.create_default_context(cafile=str(ca))) try: client.request('GET', path) response = client.getresponse() body = response.read(2 * 1024 * 1024 + 1) require(len(body) <= 2 * 1024 * 1024, 'bounded HTTPS body') return {'status': response.status, 'sha256': digest(body), 'body': body} finally: client.close() def health(): for port in (18443, 18444): deadline = time.monotonic() + 60 while True: try: result = https(port, '/healthz') require(result['status'] == 200 and result['body'] == b'ok', 'healthy exact server role') break except Exception: require(time.monotonic() < deadline, 'TLS role health deadline') time.sleep(1) auth = https(18443, '/auth/config') require(auth['status'] == 200 and json.loads(auth['body']) == {'oidc_configured': False, 'oidc_login_url': None, 'dev_login_enabled': True}, 'honest private dev/test authentication') for path in ('/', '/auth/config', '/auth/session', '/auth/dev/login', '/mcp', '/api/v1/tenants'): require(https(443, path, 'discovery.ease-technologies.de')['status'] == 404, 'public gateway has no UI/auth/MCP/admin API') require(https(443, '/api/v1/job-signing-key', 'discovery.ease-technologies.de')['status'] == 401, 'public signing key requires authenticated agent') def replace_public(path, data): require(path.parent.is_dir() and not path.is_symlink(), 'existing served directory, no symlink') next_path = path.with_name(path.name + '.script-console-' + ident) once(next_path, data, 0o644) os.replace(next_path, path) syncdir(path.parent) def after_guard(built, index): now = witness() require(now['runtime_image'] == built['image_id'], 'exact candidate remains runtime') for key in ('controls', 'pg_volume'): require(now[key] == record['baseline'][key], 'preserved post-deploy ' + key) require(now['index_sha256'] == index, 'exact post-deploy index') require(now['containers']['postgres'] == record['baseline']['containers']['postgres'], 'same PG container') for name in ('api', 'gateway'): before = dict(record['baseline']['containers'][name]) after = dict(now['containers'][name]) for key in ('id', 'image'): before.pop(key); after.pop(key) require(before == after, 'same application role, mounts, ports and closed gates') migration_guard(record, after=True) # inspect and status never mkdir, upload, launch, migrate, retag or publish. if action == 'inspect': return witness() if action == 'build': record = payload['record'] require(record['id'] == ident and record['old_image'] == payload['old_image'], 'fixed prepared case') before_guard(witness(), record['baseline']) migration_guard(record) require(record['baseline']['controls']['files']['deploy/Dockerfile'] == record['source_files']['deploy/Dockerfile'] and record['baseline']['controls']['files']['.dockerignore'] == record['source_files']['.dockerignore'] and record['baseline']['controls']['files']['deploy/server.compose.yaml'] == record['source_files']['deploy/server.compose.yaml'], 'preserving unchanged deploy controls') # Create once before the first write; an incomplete case is status-only. owned(root, True); owned(root / 'devstate', True) folder.mkdir(mode=0o700); syncdir(folder.parent) once(folder / 'intent.json', enc(record)) context = folder / 'context'; context.mkdir(mode=0o700) archive_raw = base64.b64decode(payload['source_archive'], validate=True) require(len(archive_raw) <= 20 * 1024 * 1024 and digest(archive_raw) == record['source_archive_sha256'], 'source selection archive') found = set() with tarfile.open(fileobj=io.BytesIO(archive_raw), mode='r:') as archive: for member in archive.getmembers(): name = member.name require(member.isfile() and name in record['source_files'] and name not in found and not Path(name).is_absolute() and '..' not in Path(name).parts and member.size <= 2 * 1024 * 1024, 'fixed complete source selection') data = archive.extractfile(member).read() require(digest(data) == record['source_files'][name], 'source member hash') dest = context / name; dest.parent.mkdir(mode=0o700, parents=True, exist_ok=True) once(dest, data); found.add(name) require(found == set(record['source_files']), 'complete frozen source selection') web = base64.b64decode(payload['web_archive'], validate=True) require(digest(web) == record['web_archive_sha256'] and len(web) < 2 * 1024 * 1024, 'qualified Web49 archive') once(folder / 'web.tar', web) worker = payload['worker_source'].encode() require(digest(worker) == record['worker_source_sha256'], 'fixed worker source') once(folder / 'worker.py', worker) once(folder / 'build-started.json', enc({'id': ident, 'intent_sha256': digest(enc(record))})) run(['systemd-run', '--unit=rmm-script-build-' + ident, '--property=UMask=0077', '--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'build-worker'], timeout=15) return {'id': ident, 'build_launched': True, 'deployed': False, 'lost_reply_action': 'status only'} if action == 'status': if not folder.exists(): require(not folder.is_symlink(), 'no case symlink') return {'id': ident, 'case_present': False, 'read_only': True, 'retry_permitted': False} owned(folder, True) receipts = {} for path in sorted(folder.glob('*.json')): if path.name != 'intent.json': try: receipts[path.name] = load(path) except Exception as error: receipts[path.name] = {'read_error_class': type(error).__name__} result = {'id': ident, 'case_present': True, 'receipts': receipts, 'read_only': True, 'retry_permitted': False} # Readback must survive an interrupted upload, lost launch reply, failed # migration or stopped roles. Report partial evidence; never heal it. checks = {'runtime_image': lambda: image('rust-discovery-server:local'), 'database': database, 'containers': lambda: {name: container(name) for name in ('postgres', 'api', 'gateway')}, 'controls': controls, 'index_sha256': lambda: digest(raw(root / 'web/dist/index.html'))} if (folder / 'intent.json').exists(): checks['intent_sha256'] = lambda: digest(raw(folder / 'intent.json', 8 * 1024 * 1024)) checks['source_unchanged'] = lambda: all((folder / 'context' / name).is_file() and digest(raw(folder / 'context' / name)) == value for name, value in load(folder / 'intent.json')['source_files'].items()) def served_web(): values = {} for name in load(folder / 'intent.json')['web_files']: got = https(18443, '/' + name if name != 'index.html' else '/') values[name] = {'status': got['status'], 'sha256': got['sha256']} return values checks['private_TLS_web'] = served_web for name, check in checks.items(): try: result[name] = check() except Exception as error: result[name] = {'read_error_class': type(error).__name__} for name in ('build', 'deploy'): path = folder / (name + '.log') result[name + '_log_bytes'] = path.stat().st_size if path.exists() else None return result owned(folder, True) record = load(folder / 'intent.json') require(record['id'] == ident and record['old_image'] == payload['old_image'], 'same case intent') require(digest(raw(folder / 'worker.py')) == record['worker_source_sha256'], 'retained worker unchanged') def deploy_guard(): check_source(record) before_guard(witness(), record['baseline']) migration_guard(record) built = load(folder / 'build-result.json') require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'exact successful candidate image') require(not (folder / 'deploy-started.json').exists(), 'create-once deployment; status only after any launch') return built if action == 'deploy': built = deploy_guard() once(folder / 'deploy-started.json', enc({'id': ident, 'image_id': built['image_id'], 'intent_sha256': digest(enc(record))})) run(['systemd-run', '--unit=rmm-script-deploy-' + ident, '--property=UMask=0077', '--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'deploy-worker'], timeout=15) return {'id': ident, 'deploy_launched': True, 'completed': False, 'lost_reply_action': 'status only'} # The retained worker, never caller-supplied shell code, performs each phase # once. A host-wide lock serializes this runner's build/deployment workers. require(action in ('build-worker', 'deploy-worker'), 'retained worker action') lock = os.open(root / 'devstate/.script-console-upgrade-v1.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600) fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) phase = 'build' if action == 'build-worker' else 'deploy' try: once(folder / (phase + '-worker-started.json'), enc({'id': ident})) if phase == 'build': before_guard(witness(), record['baseline']) check_source(record) require(not run(['docker', 'image', 'ls', '-q', record['candidate_tag']]).strip(), 'new candidate tag only') with (folder / 'build.log').open('xb') as log: result = subprocess.run(['docker', 'build', '--platform=linux/arm64', '--file', str(folder / 'context/deploy/Dockerfile'), '--tag', record['candidate_tag'], str(folder / 'context')], stdout=log, stderr=subprocess.STDOUT, timeout=3600) log.flush(); os.fsync(log.fileno()) require(result.returncode == 0, 'native ARM64 image build failed') check_source(record) before_guard(witness(), record['baseline']) outcome = {'passed': True, 'image_id': image(record['candidate_tag']), 'source_files_equal': True, 'runtime_unchanged': True, 'migration_executed': False, 'jobs_or_approvals_created': False} else: check_source(record) built = load(folder / 'build-result.json') require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'fixed built image') before_guard(witness(), record['baseline']) migration_guard(record) # Intent already durable. Stop both writers before checking/backup; # failures deliberately leave an incomplete case for operator readback. run(compose + ['stop', '--timeout', '20', 'api', 'gateway'], timeout=50) once(folder / '01-services-stopped.json', enc({'id': ident})) before_guard(witness(running=False), record['baseline'], stopped=True) active = sql("BEGIN READ ONLY; SELECT json_build_object('other_sessions',count(*)) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND backend_type='client backend' AND application_name<>'pg_isready'; COMMIT;") require(active['other_sessions'] == 0, 'no other database writer during upgrade') with (folder / 'postgres-before.dump').open('xb') as dump: result = subprocess.run(compose + ['exec', '-T', 'postgres', 'pg_dump', '-U', 'postgres', '-d', 'rmm', '--format=custom'], stdout=dump, stderr=subprocess.PIPE, timeout=120) dump.flush(); os.fsync(dump.fileno()) require(result.returncode == 0, 'private pre-migration database backup') once(folder / 'index-before.html', raw(root / 'web/dist/index.html')) once(folder / '02-backup-complete.json', enc({'id': ident, 'backup_private': True})) # Parse/check the entire UI archive before publishing any asset. web_raw = raw(folder / 'web.tar') require(digest(web_raw) == record['web_archive_sha256'], 'unchanged UI archive') web = {} with tarfile.open(fileobj=io.BytesIO(web_raw), mode='r:') as archive: for member in archive.getmembers(): require(member.isfile() and member.name in record['web_files'] and member.name not in web and member.size <= 2 * 1024 * 1024, 'exact qualified UI members') data = archive.extractfile(member).read() require(digest(data) == record['web_files'][member.name], 'qualified UI bytes') web[member.name] = data require(set(web) == set(record['web_files']), 'complete qualified UI') dist = root / 'web/dist'; owned(dist, True); owned(dist / 'assets', True) for name, data in web.items(): if name == 'index.html': continue dest = dist / name if dest.exists(): require(digest(raw(dest)) == record['web_files'][name], 'immutable asset filename collision') else: once(dest, data, 0o644) once(folder / '03-assets-published.json', enc({'id': ident, 'index_unchanged': digest(raw(dist / 'index.html')) == record['baseline']['index_sha256']})) require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'old index CAS before migration') require(image('rust-discovery-server:local') == record['old_image'], 'runtime tag CAS before transition') run(['docker', 'tag', built['image_id'], 'rust-discovery-server:local']) once(folder / '04-runtime-tagged.json', enc({'id': ident, 'image_id': built['image_id']})) # Named migration container remains inspectable even on failure/lost # reply. Never remove it or issue a second migration automatically. with (folder / 'deploy.log').open('xb') as log: result = subprocess.run(compose + ['run', '--name', 'rmm-script-migrate-' + ident, '--no-deps', 'migrate'], stdout=log, stderr=subprocess.STDOUT, timeout=120) log.flush(); os.fsync(log.fileno()) require(result.returncode == 0, 'migration incomplete; inspect retained named container') migration_guard(record, after=True) once(folder / '05-migration14-verified.json', enc({'id': ident, 'original_ledger_and_history_equal': True})) run(compose + ['up', '-d', '--no-deps', '--no-build', 'api', 'gateway'], timeout=60) health() after_guard(built, record['baseline']['index_sha256']) once(folder / '06-backend-ready.json', enc({'id': ident, 'TLS_healthy': True, 'public_admin_routes_closed': True})) # Directory inode and all old hashed assets survive. Only now switch # the entry document, atomically and against the exact old index. require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'final old index CAS') replace_public(dist / 'index.html', web['index.html']) for name, expected in record['web_files'].items(): served = https(18443, '/' + name if name != 'index.html' else '/') require(served['status'] == 200 and served['sha256'] == expected, 'actual private TLS UI byte hash') after_guard(built, record['web_files']['index.html']) once(folder / '07-index-published.json', enc({'id': ident, 'web_files': record['web_files']})) outcome = {'passed': True, 'image_id': built['image_id'], 'migration14': True, 'web_files': record['web_files'], 'prior_history_equal': True, 'jobs_or_approvals_created': False, 'root_system_fault_gates_closed': True, 'production_oidc_qualified': False, 'native_client_acceptance': False} once(folder / (phase + '-result.json'), enc(outcome)) return outcome except Exception as error: # No raw exception/command/DB output can escape to the caller or log. failure = {'passed': False, 'error_class': type(error).__name__, 'retry_permitted': False, 'frames': [{'function': frame.name, 'line': frame.lineno} for frame in traceback.extract_tb(error.__traceback__)]} once(folder / (phase + '-failure.json'), enc(failure)) raise RuntimeError('case incomplete; status/readback only') from None finally: os.close(lock) def archive(files): stream = io.BytesIO() with tarfile.open(fileobj=stream, mode='w:') as tar: for name, raw in sorted(files.items()): item = tarfile.TarInfo(name) item.size = len(raw) item.mode = 0o644 tar.addfile(item, io.BytesIO(raw)) return stream.getvalue() def worker_source(ident): prefix = 'import os,json,subprocess,base64,hashlib,uuid,sys\nfrom pathlib import Path\n' return (prefix + inspect.getsource(bootstrap.require) + '\n' + inspect.getsource(remote) + '\nif __name__ == "__main__":\n remote(dict(action=sys.argv[1],id=' + repr(ident) + ',old_image=' + repr(OLD_IMAGE) + '))\n').encode() def check_support(): require(all(sha((ROOT / name).read_bytes()) == expected for name, expected in SUPPORT.items()), 'pinned SSH support sources') def prepare(baseline_path): require(sha(FROZEN.read_bytes()) == FROZEN_SHA, 'frozen158 qualification manifest') frozen = json.loads(FROZEN.read_bytes()) require(frozen['source_before_after_equal'] and len(frozen['source_files']) == 158 and all(check['exit_code'] == 0 for check in frozen['checks'].values()), 'passing exact qualification') source_raw = (FROZEN.parent / 'reviewed-sources.tar.gz').read_bytes() require(sha(source_raw) == frozen['source_archive_sha256'], 'frozen source archive') sources = {} with tarfile.open(fileobj=io.BytesIO(source_raw), mode='r:gz') as tar: for item in tar.getmembers(): require(item.isfile() and item.name in frozen['source_files'] and item.name not in sources and item.size <= 2 * 1024 * 1024 and not Path(item.name).is_absolute() and '..' not in Path(item.name).parts, 'frozen source selection') data = tar.extractfile(item).read() require(sha(data) == frozen['source_files'][item.name], 'frozen member hash') sources[item.name] = data require(set(sources) == set(frozen['source_files']), 'all158 frozen source inputs retained') for name, expected in EXTRA.items(): path = ROOT / name require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'separate deployment source pin') sources[name] = path.read_bytes() web = {} for name, expected in WEB.items(): path = ROOT / 'web/dist' / name require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'qualified Web49 byte pin') web[name] = path.read_bytes() # A complete context is manifest-selected, not copied from a live checkout. for forbidden in ('credentials/', 'devstate/', '.git/', 'target/', 'web/node_modules/'): require(not any(name.startswith(forbidden) or '/' + forbidden in name for name in sources), 'private/cache path excluded') require(all(name in sources for name in ('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'crates/oidc-auth/sql/001_oidc_login_states.sql', 'tools/apt-transaction/apt_transaction.py', 'tools/apt-transaction/hook', 'deploy/linux/rmm-apt-helper.service', 'migrations/0014_windows_msi_catalog.sql')), 'Docker COPY/include closure') baseline_raw = baseline_path.read_bytes() baseline = json.loads(baseline_raw) require(baseline['runtime_image'] == OLD_IMAGE and baseline['database']['unresolved'] == 0 and [row['version'] for row in baseline['database']['migrations']] == list(range(1, 14)), 'fresh intended prior runtime/schema') ident = str(uuid.uuid4()) STATE.mkdir(mode=0o700, parents=True, exist_ok=True) folder = STATE / ident folder.mkdir(mode=0o700) data = archive(sources); web_data = archive(web); worker = worker_source(ident) checksums = {str(int(Path(name).name.split('_', 1)[0])): hashlib.sha384(value).hexdigest() for name, value in sources.items() if re.fullmatch(r'migrations/\d{4}_[a-z0-9_]+\.sql', name)} require(set(checksums) == {str(n) for n in range(1, 15)}, 'exact migration1..14 closure') record = {'schema_version': 1, 'type': 'rmm.script-console.cloud-upgrade.v1', 'id': ident, 'frozen_manifest_sha256': FROZEN_SHA, 'old_image': OLD_IMAGE, 'baseline': baseline, 'baseline_sha256': sha(baseline_raw), 'source_files': {name: sha(value) for name, value in sources.items()}, 'source_archive_sha256': sha(data), 'web_files': WEB, 'web_archive_sha256': sha(web_data), 'worker_source_sha256': sha(worker), 'runner_sha256': sha(Path(__file__).read_bytes()), 'support_sha256': SUPPORT, 'migration_checksums': checksums, 'candidate_tag': 'rust-discovery-server:script-v1-' + ident} for row in baseline['database']['migrations']: require(row['success'] and row['checksum'] == checksums[str(row['version'])], 'unchanged SQLx1..13 bytes') for name, raw in (('source.tar', data), ('web.tar', web_data), ('worker.py', worker), ('baseline.json', baseline_raw), ('manifest.json', encoded(record))): create(folder / name, raw) return {'id': ident, 'manifest': str(folder / 'manifest.json'), 'manifest_sha256': sha(encoded(record)), 'source_count': len(sources), 'remote_operations': False, 'build_started': False} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=('inspect', 'prepare', 'build', 'deploy', 'status')) parser.add_argument('--id') parser.add_argument('--baseline', type=Path) parser.add_argument('--access-root', type=Path, default=ROOT, help='existing main checkout containing protected server.conf; SSH only') parser.add_argument('--execute-cloud-build', action='store_true') parser.add_argument('--execute-cloud-deploy', action='store_true') args = parser.parse_args() os.umask(0o077) check_support() if args.action == 'prepare': require(args.baseline is not None, 'explicit saved remote baseline required') print(json.dumps(prepare(args.baseline))) return # Keep the reviewed worktree's source inputs, but reuse the existing main # checkout's protected SSH transport configuration when explicitly selected. bootstrap.ROOT = args.access_root.resolve() bootstrap.access.ROOT = bootstrap.ROOT config = bootstrap.access.load_config(bootstrap.ROOT / 'credentials/server.conf') require(config['address'] in ('88.99.80.202', 'discovery.ease-technologies.de') and config['username'] == 'root' and config.get('port', '22') == '22', 'fixed existing cloud server bootstrap target') if args.action == 'inspect': result = bootstrap.rpc('server.conf', remote, {'action': 'inspect', 'id': str(uuid.uuid4()), 'old_image': OLD_IMAGE}) require(result['runtime_image'] == OLD_IMAGE, 'known actual prior server image') STATE.mkdir(mode=0o700, parents=True, exist_ok=True) path = STATE / ('baseline-' + str(uuid.uuid4()) + '.json') create(path, encoded(result)) print(json.dumps({'baseline': str(path), 'sha256': sha(encoded(result)), 'read_only_remote': True})) return ident = str(uuid.UUID(args.id)) require(ident == args.id, 'canonical case UUID') folder = STATE / ident manifest = json.loads((folder / 'manifest.json').read_bytes()) require(manifest['id'] == ident and manifest['runner_sha256'] == sha(Path(__file__).read_bytes()) and manifest['support_sha256'] == SUPPORT and manifest['worker_source_sha256'] == sha(worker_source(ident)), 'retained exact runner/support/worker') payload = {'action': args.action, 'id': ident, 'old_image': OLD_IMAGE} if args.action in ('build', 'deploy'): require(args.execute_cloud_build if args.action == 'build' else args.execute_cloud_deploy, 'explicit bounded cloud action required') # Local once intent BEFORE SSH: uncertain/lost replies cannot launch twice. create(folder / (args.action + '-requested.json'), encoded({'id': ident, 'manifest_sha256': sha(encoded(manifest))})) if args.action == 'build': payload.update(record=manifest, source_archive=base64.b64encode((folder / 'source.tar').read_bytes()).decode(), web_archive=base64.b64encode((folder / 'web.tar').read_bytes()).decode(), worker_source=(folder / 'worker.py').read_text()) result = bootstrap.rpc('server.conf', remote, payload) create(folder / (args.action + '-reply-' + str(uuid.uuid4()) + '.json'), encoded(result)) print(json.dumps(result)) if __name__ == '__main__': try: main() except Exception as error: print('SCRIPT_CONSOLE_UPGRADE_INCOMPLETE:' + type(error).__name__ + '; preserve case, status only after any request') raise SystemExit(2) from None