Files
rust-discovery/deploy/cloud_script_console_upgrade_v1.py
T
2026-11-14 17:42:44 +01:00

693 lines
43 KiB
Python

#!/usr/bin/env python3
"""One frozen Script1.20/MSI1.19 server upgrade; no jobs, approvals or policies.
prepare is offline. inspect/status are remote read-only. build/deploy each launch
one durable, create-once case worker. Lost replies permit status only, not retry.
The notebook never runs an RMM server/client. Existing deployment tools remain
unchanged. See docs/script-console-cloud-upgrade-v1.md before actual use.
"""
import argparse
import base64
import hashlib
import inspect
import io
import json
import os
from pathlib import Path
import re
import tarfile
import uuid
import remote_client_bootstrap as bootstrap
ROOT = Path(__file__).resolve().parents[1]
STATE = ROOT / 'devstate/script-console-cloud-upgrade-v1'
FROZEN = ROOT / 'devstate/script-console-qualification/run-29461892-343f-42f4-b469-88703535425e/manifest.json'
FROZEN_SHA = 'ddd1a78d1cd289b02b1f98572f9652171ec00be84e01ce099fd722d689bcaacd'
OLD_IMAGE = 'sha256:25697f948325a11f755893e16ec5be2da138c3333c8e60f75a18f973d6ac26e2'
EXTRA = {
'deploy/Dockerfile': 'd41b4ec902c11a9e25c22b1c1d3d60d15afc207f2598ecf559cf3bdae4f8e782',
'.dockerignore': 'fdb5cba5967249f5dac35d45c5edf0be2a4e1594c94d04a2ed416d89821ace75',
'deploy/server.compose.yaml': 'd2044b0ee6d584f360990635bec149511d5a0623ec562976541900d751998818',
}
WEB = {
'index.html': '95d6f9058cc713c6af36fbd22ccc60678f081765a853b53d19ac6f0d219804f8',
'assets/index-Be1zm1OZ.css': '916e8870d4e9a9553de452b3ff6c71671bb6a7a22355de0aff432db516a9f3e7',
'assets/index-CTgvl2jr.js': 'f611662042eb4cc44f25c7d5ddc0d2d0562e32be267d3d5fb1db31b0d310f879',
}
SUPPORT = {
'deploy/remote_client_bootstrap.py': '785c43a51dfe3f93e39798dbd7611c353752ae2fc64b22e41ba168e221620d30',
'deploy/vm_access_check.py': '5c8107ea9151a9f498b2530cdc61c1a7a6d745aff853a9b05710f3395838c0dd',
}
require = bootstrap.require
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def encoded(value):
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def create(path, raw):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw)
stream.flush()
os.fsync(stream.fileno())
fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(fd)
finally:
os.close(fd)
def remote(payload):
"""Self-contained function transported by the existing strict-hostkey RPC."""
import datetime as dt
import fcntl
import http.client
import io
import platform
import socket
import ssl
import stat
import subprocess
import tarfile
import time
import traceback
os.umask(0o077)
root = Path('/opt/rust-discovery')
action = payload['action']
require(os.geteuid() == 0 and platform.machine() == 'aarch64', 'fixed root ARM64 server')
require(action in ('inspect', 'build', 'build-worker', 'deploy', 'deploy-worker', 'status'), 'fixed upgrade action')
ident = str(uuid.UUID(payload['id']))
require(ident == payload['id'], 'canonical case UUID')
folder = root / 'devstate' / ('script-console-upgrade-v1-' + ident)
compose = ['docker', 'compose', '--env-file', str(root / 'devstate/.env'),
'-f', str(root / 'deploy/server.compose.yaml')]
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def enc(value):
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def run(args, timeout=20, **kwargs):
result = subprocess.run(args, capture_output=True, timeout=timeout, **kwargs)
require(result.returncode == 0, 'fixed server command incomplete; inspect private case')
return result.stdout
def owned(path, directory=False):
value = path.lstat()
require(value.st_uid == 0 and not value.st_mode & 0o022 and not path.is_symlink()
and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'root-owned fixed path')
return value
def raw(path, limit=2 * 1024 * 1024):
value = owned(path)
require(value.st_size <= limit and value.st_nlink == 1, 'bounded private regular file')
return path.read_bytes()
def syncdir(path):
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(fd)
finally:
os.close(fd)
def once(path, data, mode=0o600):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(data)
stream.flush()
os.fchmod(stream.fileno(), mode)
os.fsync(stream.fileno())
syncdir(path.parent)
def load(path):
return json.loads(raw(path, 8 * 1024 * 1024))
def image(tag):
# Separate fields avoid ever retrieving environment variables/secrets.
value = run(['docker', 'image', 'inspect', tag, '--format', '{{.Id}} {{.Architecture}} {{.Os}}']).decode().split()
require(len(value) == 3 and value[1:] == ['arm64', 'linux'] and value[0].startswith('sha256:'), 'native ARM64 image')
return value[0]
def closed_root_policy():
# Existing MVP state: flag=true, but this exact historical policy is
# expired. Neither the flag nor the policy is changed by deployment.
directory = root / 'devstate/root-policy'
path = directory / 'policy.json'
parent = directory.lstat(); value = path.lstat()
require(not directory.is_symlink() and stat.S_ISDIR(parent.st_mode)
and parent.st_uid == 10001 and not parent.st_mode & 0o022
and not path.is_symlink() and stat.S_ISREG(value.st_mode)
and value.st_uid == 10001 and stat.S_IMODE(value.st_mode) == 0o600
and value.st_nlink == 1 and 0 < value.st_size <= 65536, 'fixed protected closed Root policy path')
data = path.read_bytes()
expected = 'cf15cbb68b590475df6def28e7dcffb7d88e0ca2d2555a06174fb654eeff05f4'
require(digest(data) == expected, 'exact historical closed Root policy hash')
policy = json.loads(data)
require(isinstance(policy, dict) and set(policy) == {'schema_version', 'profile', 'expires_at', 'scopes'}
and type(policy['schema_version']) is int and policy['schema_version'] == 1
and policy['profile'] == 'development' and isinstance(policy['expires_at'], str)
and isinstance(policy['scopes'], list) and 1 <= len(policy['scopes']) <= 64
and all(isinstance(scope, dict) for scope in policy['scopes']), 'exact legacy Root policy schema')
expiry = dt.datetime.fromisoformat(policy['expires_at'])
require(expiry.utcoffset() is not None and expiry <= dt.datetime.now(dt.timezone.utc), 'expired Root policy denies every scope')
return {'sha256': expected, 'schema_version': 1, 'profile': 'development',
'expires_at': policy['expires_at'], 'closed': True}
def container(service):
values = run(compose + ['ps', '-a', '-q', service]).decode().split()
require(len(values) == 1, 'one fixed compose service')
cid = values[0]
out = {'id': cid}
for name, fmt in (('image', '.Image'), ('user', '.Config.User'), ('running', '.State.Running'),
('mounts', '.Mounts'), ('ports', '.HostConfig.PortBindings')):
out[name] = json.loads(run(['docker', 'inspect', cid, '--format', '{{json ' + fmt + '}}']))
if service in ('api', 'gateway'):
# Filter inside Docker's template: credential env values never
# enter Python/stdout. No .env, token or private key is opened.
names = ('RMM_PROFILE', 'RMM_DEV_AUTH', 'RMM_SERVER_ROLE', 'RMM_UI_ORIGIN',
'RMM_DEV_ROOT_ENABLED', 'RMM_DEV_SYSTEM_ENABLED', 'RMM_DEV_DELIVERY_FAULTS',
'RMM_DEV_DELIVERY_FAULTS_FILE', 'RMM_DEV_SYSTEM_POLICY_FILE', 'RMM_DEV_ROOT_POLICY_FILE')
select = ' '.join('(eq $key "' + name + '")' for name in names)
fmt = '{{range .Config.Env}}{{$key := index (split . "=") 0}}{{if or ' + select + '}}{{println .}}{{end}}{{end}}'
env = dict(line.split('=', 1) for line in run(['docker', 'inspect', cid, '--format', fmt]).decode().splitlines() if '=' in line)
out['settings'] = {key: env.get(key) for key in names}
require(env.get('RMM_PROFILE') == 'development' and env.get('RMM_DEV_AUTH') == 'true'
and env.get('RMM_SERVER_ROLE') == service and env.get('RMM_UI_ORIGIN') == 'https://localhost:18443'
and env.get('RMM_DEV_ROOT_ENABLED') == 'true'
and env.get('RMM_DEV_ROOT_POLICY_FILE') == '/root-policy/policy.json'
and env.get('RMM_DEV_SYSTEM_ENABLED') is None and env.get('RMM_DEV_DELIVERY_FAULTS') == 'false'
and env.get('RMM_DEV_DELIVERY_FAULTS_FILE') is None and env.get('RMM_DEV_SYSTEM_POLICY_FILE') is None,
'unchanged private dev/test role with Root/SYSTEM/fault gates closed')
mounts = [mount for mount in out['mounts'] if mount['Destination'] == '/root-policy']
require(len(mounts) == 1 and mounts[0]['Type'] == 'bind' and mounts[0]['RW'] is False
and mounts[0]['Source'] == str(root / 'devstate/root-policy'), 'exact read-only Root policy mount')
out['closed_root_policy_sha256'] = closed_root_policy()['sha256']
require(out['user'] == '10001:10001', 'unprivileged application role')
port = '18443' if service == 'api' else '18444'
require(out['ports'] == {'18443/tcp': [{'HostIp': '127.0.0.1', 'HostPort': port}]}, 'loopback-only role')
else:
require(not out['ports'], 'PostgreSQL has no published port')
return out
def controls():
names = ('deploy/Dockerfile', '.dockerignore', 'deploy/server.compose.yaml')
files = {name: digest(raw(root / name)) for name in names}
for name in ('rmm-server-ingress.socket', 'rmm-server-ingress.service'):
files['/etc/systemd/system/' + name] = digest(raw(Path('/etc/systemd/system') / name))
# Metadata-only identity of secret trees. Private material is never read.
identities = {}
for name in ('devstate/.env', 'devstate/pki', 'devstate/api-pki', 'devstate/jobs', 'devstate/root-policy'):
path = root / name
paths = [path, *sorted(path.rglob('*'))] if path.is_dir() else [path]
for item in paths:
value = item.lstat()
require(not item.is_symlink() and (stat.S_ISREG(value.st_mode) or stat.S_ISDIR(value.st_mode)), 'preserved state object')
identities[str(item.relative_to(root))] = [value.st_dev, value.st_ino, value.st_uid, value.st_gid,
value.st_mode, value.st_size if stat.S_ISREG(value.st_mode) else None,
value.st_mtime_ns if stat.S_ISREG(value.st_mode) else None]
ingress = run(['systemctl', 'show', 'rmm-server-ingress.socket', '-p', 'ActiveState', '-p', 'UnitFileState']).decode()
require('ActiveState=active' in ingress, 'existing public socket ingress stays active')
return {'files': files, 'protected_metadata': identities, 'ingress': ingress,
'root_policy': closed_root_policy()}
def sql(query):
return json.loads(run(compose + ['exec', '-T', 'postgres', 'psql', '-X', '-qAt', '-v',
'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm'], timeout=30, input=query.encode()))
def database():
# One repeatable-read snapshot, only counts and SHA256 values leave PG.
tables = ('tenants', 'assets', 'operation_plans', 'approval_requests', 'authorizations', 'operations',
'jobs', 'outbox', 'execution_permits', 'job_receipts', 'audit_events', 'operation_submissions',
'live_query_requests', 'memberships', 'tenant_limits', 'oidc_principals', 'package_recipes', 'artifacts')
pairs = []
for table in tables:
pairs.append("'" + table + "',(SELECT json_build_object('count',count(*),'sha256',encode(sha256(convert_to("
"coalesce(string_agg(to_jsonb(t)::text,E'\\n' ORDER BY to_jsonb(t)::text),''),'UTF8')),'hex')) FROM " + table + ' t)')
# Stable device identity excludes expected changing passive Sync/contact.
identity = "(SELECT coalesce(json_agg(q ORDER BY tenant_id,agent_id),'[]'::json) FROM (SELECT tenant_id,agent_id,asset_id,enrollment_generation,assignment_version,certificate_sha256,revoked_at FROM agent_installations) q)"
migrations = "(SELECT json_agg(q ORDER BY version) FROM (SELECT version,description,installed_on,success,encode(checksum,'hex') AS checksum,execution_time FROM _sqlx_migrations) q)"
role = "(SELECT json_build_object('superuser',rolsuper,'bypassrls',rolbypassrls) FROM pg_roles WHERE rolname='rmm_app')"
query = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SELECT json_build_object('history',json_build_object(" + ','.join(pairs) + "),'identities'," + identity + ",'migrations'," + migrations + ",'app_role'," + role + ",'unresolved',(SELECT count(*) FROM jobs WHERE state NOT IN ('succeeded','failed','cancelled','expired'))); COMMIT;"
out = sql(query)
require(out['unresolved'] == 0 and out['app_role'] == {'superuser': False, 'bypassrls': False}, 'settled history and no RLS bypass')
require(all(row['success'] for row in out['migrations']), 'successful SQLx ledger')
return out
def witness(running=True):
current = {'runtime_image': image('rust-discovery-server:local'),
'controls': controls(), 'containers': {name: container(name) for name in ('postgres', 'api', 'gateway')},
'index_sha256': digest(raw(root / 'web/dist/index.html')), 'database': database()}
pg = current['containers']['postgres']
mounts = [m for m in pg['mounts'] if m['Destination'] == '/var/lib/postgresql/data']
require(len(mounts) == 1 and mounts[0]['Type'] == 'volume' and pg['running'], 'existing PG data volume')
value = Path(mounts[0]['Source']).lstat()
require(stat.S_ISDIR(value.st_mode) and not Path(mounts[0]['Source']).is_symlink(), 'same PG data directory')
current['pg_volume'] = {'mount': mounts[0], 'device': value.st_dev, 'inode': value.st_ino}
for name in ('api', 'gateway'):
require(current['containers'][name]['running'] == running, 'fixed application running state')
require(current['containers'][name]['image'] == current['runtime_image'], 'both services run runtime tag')
return current
def before_guard(now, expected, stopped=False):
require(now['runtime_image'] == expected['runtime_image'] == payload['old_image'], 'runtime image CAS')
for key in ('controls', 'index_sha256', 'database', 'pg_volume'):
require(now[key] == expected[key], 'preserved ' + key + ' CAS')
for name in ('postgres', 'api', 'gateway'):
current = dict(now['containers'][name])
old = dict(expected['containers'][name])
if stopped and name != 'postgres':
current.pop('running'); old.pop('running')
require(current == old, 'fixed prior service/container CAS')
def migration_guard(record, after=False):
expected = record['migration_checksums']
require([row['version'] for row in record['baseline']['database']['migrations']] == list(range(1, 14)), 'exact prior migrations 1..13')
for row in record['baseline']['database']['migrations']:
require(row['checksum'] == expected[str(row['version'])], 'frozen old migration checksum')
if after:
now = database()
require(now['migrations'][:13] == record['baseline']['database']['migrations']
and [row['version'] for row in now['migrations']] == list(range(1, 15))
and now['migrations'][13]['checksum'] == expected['14'], 'only additive frozen migration14')
for key in ('history', 'identities', 'app_role', 'unresolved'):
require(now[key] == record['baseline']['database'][key], 'original database rows unchanged')
check = sql("BEGIN READ ONLY; SELECT json_build_object('count',(SELECT count(*) FROM windows_msi_recipes),"
"'rls',(SELECT relrowsecurity AND relforcerowsecurity FROM pg_class WHERE oid='windows_msi_recipes'::regclass),"
"'select',has_table_privilege('rmm_app','windows_msi_recipes','SELECT'),"
"'insert',has_table_privilege('rmm_app','windows_msi_recipes','INSERT'),"
"'update',has_table_privilege('rmm_app','windows_msi_recipes','UPDATE'),"
"'delete',has_table_privilege('rmm_app','windows_msi_recipes','DELETE'),"
"'trigger',EXISTS(SELECT 1 FROM pg_trigger WHERE tgrelid='windows_msi_recipes'::regclass AND tgname='immutable_content' AND tgenabled='O')); COMMIT;")
require(check == {'count': 0, 'rls': True, 'select': True, 'insert': True, 'update': False, 'delete': False, 'trigger': True}, 'empty immutable MSI tenant catalog with FORCE RLS')
return now
def check_source(record):
require(all(digest(raw(folder / 'context' / name)) == value for name, value in record['source_files'].items()), 'frozen build context unchanged')
def https(port, path, host='localhost'):
require((port, host) in ((18443, 'localhost'), (18444, 'localhost'),
(443, 'discovery.ease-technologies.de')), 'fixed role TLS endpoint and name')
ca = root / ('devstate/api-pki/ca.pem' if port == 18443 else 'devstate/pki/ca.pem')
class HTTPS(http.client.HTTPSConnection):
def connect(self):
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', port), timeout=5), server_hostname=self.host)
client = HTTPS(host, port, timeout=5, context=ssl.create_default_context(cafile=str(ca)))
try:
client.request('GET', path)
response = client.getresponse()
body = response.read(2 * 1024 * 1024 + 1)
require(len(body) <= 2 * 1024 * 1024, 'bounded HTTPS body')
return {'status': response.status, 'sha256': digest(body), 'body': body}
finally:
client.close()
def health():
for port in (18443, 18444):
deadline = time.monotonic() + 60
while True:
try:
result = https(port, '/healthz')
require(result['status'] == 200 and result['body'] == b'ok', 'healthy exact server role')
break
except Exception:
require(time.monotonic() < deadline, 'TLS role health deadline')
time.sleep(1)
auth = https(18443, '/auth/config')
require(auth['status'] == 200 and json.loads(auth['body']) ==
{'oidc_configured': False, 'oidc_login_url': None, 'dev_login_enabled': True}, 'honest private dev/test authentication')
for path in ('/', '/auth/config', '/auth/session', '/auth/dev/login', '/mcp', '/api/v1/tenants'):
require(https(443, path, 'discovery.ease-technologies.de')['status'] == 404, 'public gateway has no UI/auth/MCP/admin API')
require(https(443, '/api/v1/job-signing-key', 'discovery.ease-technologies.de')['status'] == 401, 'public signing key requires authenticated agent')
def replace_public(path, data):
require(path.parent.is_dir() and not path.is_symlink(), 'existing served directory, no symlink')
next_path = path.with_name(path.name + '.script-console-' + ident)
once(next_path, data, 0o644)
os.replace(next_path, path)
syncdir(path.parent)
def after_guard(built, index):
now = witness()
require(now['runtime_image'] == built['image_id'], 'exact candidate remains runtime')
for key in ('controls', 'pg_volume'):
require(now[key] == record['baseline'][key], 'preserved post-deploy ' + key)
require(now['index_sha256'] == index, 'exact post-deploy index')
require(now['containers']['postgres'] == record['baseline']['containers']['postgres'], 'same PG container')
for name in ('api', 'gateway'):
before = dict(record['baseline']['containers'][name])
after = dict(now['containers'][name])
for key in ('id', 'image'):
before.pop(key); after.pop(key)
require(before == after, 'same application role, mounts, ports and closed gates')
migration_guard(record, after=True)
# inspect and status never mkdir, upload, launch, migrate, retag or publish.
if action == 'inspect':
return witness()
if action == 'build':
record = payload['record']
require(record['id'] == ident and record['old_image'] == payload['old_image'], 'fixed prepared case')
before_guard(witness(), record['baseline'])
migration_guard(record)
require(record['baseline']['controls']['files']['deploy/Dockerfile'] == record['source_files']['deploy/Dockerfile']
and record['baseline']['controls']['files']['.dockerignore'] == record['source_files']['.dockerignore']
and record['baseline']['controls']['files']['deploy/server.compose.yaml'] == record['source_files']['deploy/server.compose.yaml'], 'preserving unchanged deploy controls')
# Create once before the first write; an incomplete case is status-only.
owned(root, True); owned(root / 'devstate', True)
folder.mkdir(mode=0o700); syncdir(folder.parent)
once(folder / 'intent.json', enc(record))
context = folder / 'context'; context.mkdir(mode=0o700)
archive_raw = base64.b64decode(payload['source_archive'], validate=True)
require(len(archive_raw) <= 20 * 1024 * 1024 and digest(archive_raw) == record['source_archive_sha256'], 'source selection archive')
found = set()
with tarfile.open(fileobj=io.BytesIO(archive_raw), mode='r:') as archive:
for member in archive.getmembers():
name = member.name
require(member.isfile() and name in record['source_files'] and name not in found
and not Path(name).is_absolute() and '..' not in Path(name).parts
and member.size <= 2 * 1024 * 1024, 'fixed complete source selection')
data = archive.extractfile(member).read()
require(digest(data) == record['source_files'][name], 'source member hash')
dest = context / name; dest.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
once(dest, data); found.add(name)
require(found == set(record['source_files']), 'complete frozen source selection')
web = base64.b64decode(payload['web_archive'], validate=True)
require(digest(web) == record['web_archive_sha256'] and len(web) < 2 * 1024 * 1024, 'qualified Web49 archive')
once(folder / 'web.tar', web)
worker = payload['worker_source'].encode()
require(digest(worker) == record['worker_source_sha256'], 'fixed worker source')
once(folder / 'worker.py', worker)
once(folder / 'build-started.json', enc({'id': ident, 'intent_sha256': digest(enc(record))}))
run(['systemd-run', '--unit=rmm-script-build-' + ident, '--property=UMask=0077',
'--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'build-worker'], timeout=15)
return {'id': ident, 'build_launched': True, 'deployed': False, 'lost_reply_action': 'status only'}
if action == 'status':
if not folder.exists():
require(not folder.is_symlink(), 'no case symlink')
return {'id': ident, 'case_present': False, 'read_only': True, 'retry_permitted': False}
owned(folder, True)
receipts = {}
for path in sorted(folder.glob('*.json')):
if path.name != 'intent.json':
try:
receipts[path.name] = load(path)
except Exception as error:
receipts[path.name] = {'read_error_class': type(error).__name__}
result = {'id': ident, 'case_present': True, 'receipts': receipts, 'read_only': True, 'retry_permitted': False}
# Readback must survive an interrupted upload, lost launch reply, failed
# migration or stopped roles. Report partial evidence; never heal it.
checks = {'runtime_image': lambda: image('rust-discovery-server:local'), 'database': database,
'containers': lambda: {name: container(name) for name in ('postgres', 'api', 'gateway')},
'controls': controls, 'index_sha256': lambda: digest(raw(root / 'web/dist/index.html'))}
if (folder / 'intent.json').exists():
checks['intent_sha256'] = lambda: digest(raw(folder / 'intent.json', 8 * 1024 * 1024))
checks['source_unchanged'] = lambda: all((folder / 'context' / name).is_file()
and digest(raw(folder / 'context' / name)) == value
for name, value in load(folder / 'intent.json')['source_files'].items())
def served_web():
values = {}
for name in load(folder / 'intent.json')['web_files']:
got = https(18443, '/' + name if name != 'index.html' else '/')
values[name] = {'status': got['status'], 'sha256': got['sha256']}
return values
checks['private_TLS_web'] = served_web
for name, check in checks.items():
try:
result[name] = check()
except Exception as error:
result[name] = {'read_error_class': type(error).__name__}
for name in ('build', 'deploy'):
path = folder / (name + '.log')
result[name + '_log_bytes'] = path.stat().st_size if path.exists() else None
return result
owned(folder, True)
record = load(folder / 'intent.json')
require(record['id'] == ident and record['old_image'] == payload['old_image'], 'same case intent')
require(digest(raw(folder / 'worker.py')) == record['worker_source_sha256'], 'retained worker unchanged')
def deploy_guard():
check_source(record)
before_guard(witness(), record['baseline'])
migration_guard(record)
built = load(folder / 'build-result.json')
require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'exact successful candidate image')
require(not (folder / 'deploy-started.json').exists(), 'create-once deployment; status only after any launch')
return built
if action == 'deploy':
built = deploy_guard()
once(folder / 'deploy-started.json', enc({'id': ident, 'image_id': built['image_id'], 'intent_sha256': digest(enc(record))}))
run(['systemd-run', '--unit=rmm-script-deploy-' + ident, '--property=UMask=0077',
'--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'deploy-worker'], timeout=15)
return {'id': ident, 'deploy_launched': True, 'completed': False, 'lost_reply_action': 'status only'}
# The retained worker, never caller-supplied shell code, performs each phase
# once. A host-wide lock serializes this runner's build/deployment workers.
require(action in ('build-worker', 'deploy-worker'), 'retained worker action')
lock = os.open(root / 'devstate/.script-console-upgrade-v1.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
phase = 'build' if action == 'build-worker' else 'deploy'
try:
once(folder / (phase + '-worker-started.json'), enc({'id': ident}))
if phase == 'build':
before_guard(witness(), record['baseline'])
check_source(record)
require(not run(['docker', 'image', 'ls', '-q', record['candidate_tag']]).strip(), 'new candidate tag only')
with (folder / 'build.log').open('xb') as log:
result = subprocess.run(['docker', 'build', '--platform=linux/arm64', '--file', str(folder / 'context/deploy/Dockerfile'),
'--tag', record['candidate_tag'], str(folder / 'context')], stdout=log, stderr=subprocess.STDOUT, timeout=3600)
log.flush(); os.fsync(log.fileno())
require(result.returncode == 0, 'native ARM64 image build failed')
check_source(record)
before_guard(witness(), record['baseline'])
outcome = {'passed': True, 'image_id': image(record['candidate_tag']), 'source_files_equal': True,
'runtime_unchanged': True, 'migration_executed': False, 'jobs_or_approvals_created': False}
else:
check_source(record)
built = load(folder / 'build-result.json')
require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'fixed built image')
before_guard(witness(), record['baseline'])
migration_guard(record)
# Intent already durable. Stop both writers before checking/backup;
# failures deliberately leave an incomplete case for operator readback.
run(compose + ['stop', '--timeout', '20', 'api', 'gateway'], timeout=50)
once(folder / '01-services-stopped.json', enc({'id': ident}))
before_guard(witness(running=False), record['baseline'], stopped=True)
active = sql("BEGIN READ ONLY; SELECT json_build_object('other_sessions',count(*)) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND backend_type='client backend' AND application_name<>'pg_isready'; COMMIT;")
require(active['other_sessions'] == 0, 'no other database writer during upgrade')
with (folder / 'postgres-before.dump').open('xb') as dump:
result = subprocess.run(compose + ['exec', '-T', 'postgres', 'pg_dump', '-U', 'postgres', '-d', 'rmm', '--format=custom'],
stdout=dump, stderr=subprocess.PIPE, timeout=120)
dump.flush(); os.fsync(dump.fileno())
require(result.returncode == 0, 'private pre-migration database backup')
once(folder / 'index-before.html', raw(root / 'web/dist/index.html'))
once(folder / '02-backup-complete.json', enc({'id': ident, 'backup_private': True}))
# Parse/check the entire UI archive before publishing any asset.
web_raw = raw(folder / 'web.tar')
require(digest(web_raw) == record['web_archive_sha256'], 'unchanged UI archive')
web = {}
with tarfile.open(fileobj=io.BytesIO(web_raw), mode='r:') as archive:
for member in archive.getmembers():
require(member.isfile() and member.name in record['web_files'] and member.name not in web
and member.size <= 2 * 1024 * 1024, 'exact qualified UI members')
data = archive.extractfile(member).read()
require(digest(data) == record['web_files'][member.name], 'qualified UI bytes')
web[member.name] = data
require(set(web) == set(record['web_files']), 'complete qualified UI')
dist = root / 'web/dist'; owned(dist, True); owned(dist / 'assets', True)
for name, data in web.items():
if name == 'index.html':
continue
dest = dist / name
if dest.exists():
require(digest(raw(dest)) == record['web_files'][name], 'immutable asset filename collision')
else:
once(dest, data, 0o644)
once(folder / '03-assets-published.json', enc({'id': ident, 'index_unchanged': digest(raw(dist / 'index.html')) == record['baseline']['index_sha256']}))
require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'old index CAS before migration')
require(image('rust-discovery-server:local') == record['old_image'], 'runtime tag CAS before transition')
run(['docker', 'tag', built['image_id'], 'rust-discovery-server:local'])
once(folder / '04-runtime-tagged.json', enc({'id': ident, 'image_id': built['image_id']}))
# Named migration container remains inspectable even on failure/lost
# reply. Never remove it or issue a second migration automatically.
with (folder / 'deploy.log').open('xb') as log:
result = subprocess.run(compose + ['run', '--name', 'rmm-script-migrate-' + ident,
'--no-deps', 'migrate'], stdout=log, stderr=subprocess.STDOUT, timeout=120)
log.flush(); os.fsync(log.fileno())
require(result.returncode == 0, 'migration incomplete; inspect retained named container')
migration_guard(record, after=True)
once(folder / '05-migration14-verified.json', enc({'id': ident, 'original_ledger_and_history_equal': True}))
run(compose + ['up', '-d', '--no-deps', '--no-build', 'api', 'gateway'], timeout=60)
health()
after_guard(built, record['baseline']['index_sha256'])
once(folder / '06-backend-ready.json', enc({'id': ident, 'TLS_healthy': True, 'public_admin_routes_closed': True}))
# Directory inode and all old hashed assets survive. Only now switch
# the entry document, atomically and against the exact old index.
require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'final old index CAS')
replace_public(dist / 'index.html', web['index.html'])
for name, expected in record['web_files'].items():
served = https(18443, '/' + name if name != 'index.html' else '/')
require(served['status'] == 200 and served['sha256'] == expected, 'actual private TLS UI byte hash')
after_guard(built, record['web_files']['index.html'])
once(folder / '07-index-published.json', enc({'id': ident, 'web_files': record['web_files']}))
outcome = {'passed': True, 'image_id': built['image_id'], 'migration14': True, 'web_files': record['web_files'],
'prior_history_equal': True, 'jobs_or_approvals_created': False, 'root_system_fault_gates_closed': True,
'production_oidc_qualified': False, 'native_client_acceptance': False}
once(folder / (phase + '-result.json'), enc(outcome))
return outcome
except Exception as error:
# No raw exception/command/DB output can escape to the caller or log.
failure = {'passed': False, 'error_class': type(error).__name__, 'retry_permitted': False,
'frames': [{'function': frame.name, 'line': frame.lineno} for frame in traceback.extract_tb(error.__traceback__)]}
once(folder / (phase + '-failure.json'), enc(failure))
raise RuntimeError('case incomplete; status/readback only') from None
finally:
os.close(lock)
def archive(files):
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode='w:') as tar:
for name, raw in sorted(files.items()):
item = tarfile.TarInfo(name)
item.size = len(raw)
item.mode = 0o644
tar.addfile(item, io.BytesIO(raw))
return stream.getvalue()
def worker_source(ident):
prefix = 'import os,json,subprocess,base64,hashlib,uuid,sys\nfrom pathlib import Path\n'
return (prefix + inspect.getsource(bootstrap.require) + '\n' + inspect.getsource(remote) +
'\nif __name__ == "__main__":\n remote(dict(action=sys.argv[1],id=' + repr(ident) + ',old_image=' + repr(OLD_IMAGE) + '))\n').encode()
def check_support():
require(all(sha((ROOT / name).read_bytes()) == expected for name, expected in SUPPORT.items()), 'pinned SSH support sources')
def prepare(baseline_path):
require(sha(FROZEN.read_bytes()) == FROZEN_SHA, 'frozen158 qualification manifest')
frozen = json.loads(FROZEN.read_bytes())
require(frozen['source_before_after_equal'] and len(frozen['source_files']) == 158
and all(check['exit_code'] == 0 for check in frozen['checks'].values()), 'passing exact qualification')
source_raw = (FROZEN.parent / 'reviewed-sources.tar.gz').read_bytes()
require(sha(source_raw) == frozen['source_archive_sha256'], 'frozen source archive')
sources = {}
with tarfile.open(fileobj=io.BytesIO(source_raw), mode='r:gz') as tar:
for item in tar.getmembers():
require(item.isfile() and item.name in frozen['source_files'] and item.name not in sources
and item.size <= 2 * 1024 * 1024 and not Path(item.name).is_absolute()
and '..' not in Path(item.name).parts, 'frozen source selection')
data = tar.extractfile(item).read()
require(sha(data) == frozen['source_files'][item.name], 'frozen member hash')
sources[item.name] = data
require(set(sources) == set(frozen['source_files']), 'all158 frozen source inputs retained')
for name, expected in EXTRA.items():
path = ROOT / name
require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'separate deployment source pin')
sources[name] = path.read_bytes()
web = {}
for name, expected in WEB.items():
path = ROOT / 'web/dist' / name
require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'qualified Web49 byte pin')
web[name] = path.read_bytes()
# A complete context is manifest-selected, not copied from a live checkout.
for forbidden in ('credentials/', 'devstate/', '.git/', 'target/', 'web/node_modules/'):
require(not any(name.startswith(forbidden) or '/' + forbidden in name for name in sources), 'private/cache path excluded')
require(all(name in sources for name in ('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml',
'crates/oidc-auth/sql/001_oidc_login_states.sql', 'tools/apt-transaction/apt_transaction.py',
'tools/apt-transaction/hook', 'deploy/linux/rmm-apt-helper.service', 'migrations/0014_windows_msi_catalog.sql')), 'Docker COPY/include closure')
baseline_raw = baseline_path.read_bytes()
baseline = json.loads(baseline_raw)
require(baseline['runtime_image'] == OLD_IMAGE and baseline['database']['unresolved'] == 0
and [row['version'] for row in baseline['database']['migrations']] == list(range(1, 14)), 'fresh intended prior runtime/schema')
ident = str(uuid.uuid4())
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
folder = STATE / ident
folder.mkdir(mode=0o700)
data = archive(sources); web_data = archive(web); worker = worker_source(ident)
checksums = {str(int(Path(name).name.split('_', 1)[0])): hashlib.sha384(value).hexdigest()
for name, value in sources.items() if re.fullmatch(r'migrations/\d{4}_[a-z0-9_]+\.sql', name)}
require(set(checksums) == {str(n) for n in range(1, 15)}, 'exact migration1..14 closure')
record = {'schema_version': 1, 'type': 'rmm.script-console.cloud-upgrade.v1', 'id': ident,
'frozen_manifest_sha256': FROZEN_SHA, 'old_image': OLD_IMAGE, 'baseline': baseline,
'baseline_sha256': sha(baseline_raw), 'source_files': {name: sha(value) for name, value in sources.items()},
'source_archive_sha256': sha(data), 'web_files': WEB, 'web_archive_sha256': sha(web_data),
'worker_source_sha256': sha(worker), 'runner_sha256': sha(Path(__file__).read_bytes()),
'support_sha256': SUPPORT, 'migration_checksums': checksums,
'candidate_tag': 'rust-discovery-server:script-v1-' + ident}
for row in baseline['database']['migrations']:
require(row['success'] and row['checksum'] == checksums[str(row['version'])], 'unchanged SQLx1..13 bytes')
for name, raw in (('source.tar', data), ('web.tar', web_data), ('worker.py', worker),
('baseline.json', baseline_raw), ('manifest.json', encoded(record))):
create(folder / name, raw)
return {'id': ident, 'manifest': str(folder / 'manifest.json'), 'manifest_sha256': sha(encoded(record)),
'source_count': len(sources), 'remote_operations': False, 'build_started': False}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('inspect', 'prepare', 'build', 'deploy', 'status'))
parser.add_argument('--id')
parser.add_argument('--baseline', type=Path)
parser.add_argument('--access-root', type=Path, default=ROOT,
help='existing main checkout containing protected server.conf; SSH only')
parser.add_argument('--execute-cloud-build', action='store_true')
parser.add_argument('--execute-cloud-deploy', action='store_true')
args = parser.parse_args()
os.umask(0o077)
check_support()
if args.action == 'prepare':
require(args.baseline is not None, 'explicit saved remote baseline required')
print(json.dumps(prepare(args.baseline)))
return
# Keep the reviewed worktree's source inputs, but reuse the existing main
# checkout's protected SSH transport configuration when explicitly selected.
bootstrap.ROOT = args.access_root.resolve()
bootstrap.access.ROOT = bootstrap.ROOT
config = bootstrap.access.load_config(bootstrap.ROOT / 'credentials/server.conf')
require(config['address'] in ('88.99.80.202', 'discovery.ease-technologies.de')
and config['username'] == 'root' and config.get('port', '22') == '22', 'fixed existing cloud server bootstrap target')
if args.action == 'inspect':
result = bootstrap.rpc('server.conf', remote, {'action': 'inspect', 'id': str(uuid.uuid4()), 'old_image': OLD_IMAGE})
require(result['runtime_image'] == OLD_IMAGE, 'known actual prior server image')
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
path = STATE / ('baseline-' + str(uuid.uuid4()) + '.json')
create(path, encoded(result))
print(json.dumps({'baseline': str(path), 'sha256': sha(encoded(result)), 'read_only_remote': True}))
return
ident = str(uuid.UUID(args.id))
require(ident == args.id, 'canonical case UUID')
folder = STATE / ident
manifest = json.loads((folder / 'manifest.json').read_bytes())
require(manifest['id'] == ident and manifest['runner_sha256'] == sha(Path(__file__).read_bytes())
and manifest['support_sha256'] == SUPPORT and manifest['worker_source_sha256'] == sha(worker_source(ident)), 'retained exact runner/support/worker')
payload = {'action': args.action, 'id': ident, 'old_image': OLD_IMAGE}
if args.action in ('build', 'deploy'):
require(args.execute_cloud_build if args.action == 'build' else args.execute_cloud_deploy, 'explicit bounded cloud action required')
# Local once intent BEFORE SSH: uncertain/lost replies cannot launch twice.
create(folder / (args.action + '-requested.json'), encoded({'id': ident, 'manifest_sha256': sha(encoded(manifest))}))
if args.action == 'build':
payload.update(record=manifest, source_archive=base64.b64encode((folder / 'source.tar').read_bytes()).decode(),
web_archive=base64.b64encode((folder / 'web.tar').read_bytes()).decode(), worker_source=(folder / 'worker.py').read_text())
result = bootstrap.rpc('server.conf', remote, payload)
create(folder / (args.action + '-reply-' + str(uuid.uuid4()) + '.json'), encoded(result))
print(json.dumps(result))
if __name__ == '__main__':
try:
main()
except Exception as error:
print('SCRIPT_CONSOLE_UPGRADE_INCOMPLETE:' + type(error).__name__ + '; preserve case, status only after any request')
raise SystemExit(2) from None