712 lines
44 KiB
Python
712 lines
44 KiB
Python
#!/usr/bin/env python3
|
|
"""Version2: exact preserved cloud upgrade, with deterministic Docker mounts.
|
|
|
|
prepare is offline. inspect/status are remote read-only. build/deploy each launch
|
|
one durable, create-once case worker. Lost replies permit status only, not retry.
|
|
The notebook never runs an RMM server/client. Existing deployment tools remain
|
|
unchanged. See docs/script-console-cloud-upgrade-v2.md before actual use.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import inspect
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import tarfile
|
|
import uuid
|
|
|
|
import remote_client_bootstrap as bootstrap
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
STATE = ROOT / 'devstate/script-console-cloud-upgrade-v2'
|
|
FROZEN = ROOT / 'devstate/script-console-qualification/run-29461892-343f-42f4-b469-88703535425e/manifest.json'
|
|
FROZEN_SHA = 'ddd1a78d1cd289b02b1f98572f9652171ec00be84e01ce099fd722d689bcaacd'
|
|
OLD_IMAGE = 'sha256:25697f948325a11f755893e16ec5be2da138c3333c8e60f75a18f973d6ac26e2'
|
|
EXTRA = {
|
|
'deploy/Dockerfile': 'd41b4ec902c11a9e25c22b1c1d3d60d15afc207f2598ecf559cf3bdae4f8e782',
|
|
'.dockerignore': 'fdb5cba5967249f5dac35d45c5edf0be2a4e1594c94d04a2ed416d89821ace75',
|
|
'deploy/server.compose.yaml': 'd2044b0ee6d584f360990635bec149511d5a0623ec562976541900d751998818',
|
|
}
|
|
WEB = {
|
|
'index.html': '95d6f9058cc713c6af36fbd22ccc60678f081765a853b53d19ac6f0d219804f8',
|
|
'assets/index-Be1zm1OZ.css': '916e8870d4e9a9553de452b3ff6c71671bb6a7a22355de0aff432db516a9f3e7',
|
|
'assets/index-CTgvl2jr.js': 'f611662042eb4cc44f25c7d5ddc0d2d0562e32be267d3d5fb1db31b0d310f879',
|
|
}
|
|
SUPPORT = {
|
|
'deploy/remote_client_bootstrap.py': '785c43a51dfe3f93e39798dbd7611c353752ae2fc64b22e41ba168e221620d30',
|
|
'deploy/vm_access_check.py': '5c8107ea9151a9f498b2530cdc61c1a7a6d745aff853a9b05710f3395838c0dd',
|
|
}
|
|
PREVIOUS_FAILED_CASE = {
|
|
'id': '499e13fb-7433-405d-9a98-0c27ca864246',
|
|
'manifest_sha256': '305de92d11b90184589210fd2b0f301ecaf5d9a75467bff5e81c8518588ad416',
|
|
'worker_sha256': '6d056e6385ba67c00414de3dc626a51b4f022947bee6381d92908acce3a2b4a9',
|
|
'readback_sha256': 'ed2fcd82cdca7692d9c5d122e4cccc1b5aa5a5dc6fec4ac9035ecef647b8e8ef',
|
|
'failure_before_docker_build': True,
|
|
}
|
|
require = bootstrap.require
|
|
|
|
|
|
def sha(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def encoded(value):
|
|
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
|
|
|
|
|
|
def create(path, raw):
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'wb') as stream:
|
|
stream.write(raw)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def remote(payload):
|
|
"""Self-contained function transported by the existing strict-hostkey RPC."""
|
|
import datetime as dt
|
|
import fcntl
|
|
import http.client
|
|
import io
|
|
import platform
|
|
import socket
|
|
import ssl
|
|
import stat
|
|
import subprocess
|
|
import tarfile
|
|
import time
|
|
import traceback
|
|
os.umask(0o077)
|
|
root = Path('/opt/rust-discovery')
|
|
action = payload['action']
|
|
require(os.geteuid() == 0 and platform.machine() == 'aarch64', 'fixed root ARM64 server')
|
|
require(action in ('inspect', 'build', 'build-worker', 'deploy', 'deploy-worker', 'status'), 'fixed upgrade action')
|
|
ident = str(uuid.UUID(payload['id']))
|
|
require(ident == payload['id'], 'canonical case UUID')
|
|
folder = root / 'devstate' / ('script-console-upgrade-v2-' + ident)
|
|
compose = ['docker', 'compose', '--env-file', str(root / 'devstate/.env'),
|
|
'-f', str(root / 'deploy/server.compose.yaml')]
|
|
|
|
def digest(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
def enc(value):
|
|
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
|
|
|
|
def run(args, timeout=20, **kwargs):
|
|
result = subprocess.run(args, capture_output=True, timeout=timeout, **kwargs)
|
|
require(result.returncode == 0, 'fixed server command incomplete; inspect private case')
|
|
return result.stdout
|
|
|
|
def owned(path, directory=False):
|
|
value = path.lstat()
|
|
require(value.st_uid == 0 and not value.st_mode & 0o022 and not path.is_symlink()
|
|
and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'root-owned fixed path')
|
|
return value
|
|
|
|
def raw(path, limit=2 * 1024 * 1024):
|
|
value = owned(path)
|
|
require(value.st_size <= limit and value.st_nlink == 1, 'bounded private regular file')
|
|
return path.read_bytes()
|
|
|
|
def syncdir(path):
|
|
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
def once(path, data, mode=0o600):
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
|
|
with os.fdopen(fd, 'wb') as stream:
|
|
stream.write(data)
|
|
stream.flush()
|
|
os.fchmod(stream.fileno(), mode)
|
|
os.fsync(stream.fileno())
|
|
syncdir(path.parent)
|
|
|
|
def load(path):
|
|
return json.loads(raw(path, 8 * 1024 * 1024))
|
|
|
|
def image(tag):
|
|
# Separate fields avoid ever retrieving environment variables/secrets.
|
|
value = run(['docker', 'image', 'inspect', tag, '--format', '{{.Id}} {{.Architecture}} {{.Os}}']).decode().split()
|
|
require(len(value) == 3 and value[1:] == ['arm64', 'linux'] and value[0].startswith('sha256:'), 'native ARM64 image')
|
|
return value[0]
|
|
|
|
def closed_root_policy():
|
|
# Existing MVP state: flag=true, but this exact historical policy is
|
|
# expired. Neither the flag nor the policy is changed by deployment.
|
|
directory = root / 'devstate/root-policy'
|
|
path = directory / 'policy.json'
|
|
parent = directory.lstat(); value = path.lstat()
|
|
require(not directory.is_symlink() and stat.S_ISDIR(parent.st_mode)
|
|
and parent.st_uid == 10001 and not parent.st_mode & 0o022
|
|
and not path.is_symlink() and stat.S_ISREG(value.st_mode)
|
|
and value.st_uid == 10001 and stat.S_IMODE(value.st_mode) == 0o600
|
|
and value.st_nlink == 1 and 0 < value.st_size <= 65536, 'fixed protected closed Root policy path')
|
|
data = path.read_bytes()
|
|
expected = 'cf15cbb68b590475df6def28e7dcffb7d88e0ca2d2555a06174fb654eeff05f4'
|
|
require(digest(data) == expected, 'exact historical closed Root policy hash')
|
|
policy = json.loads(data)
|
|
require(isinstance(policy, dict) and set(policy) == {'schema_version', 'profile', 'expires_at', 'scopes'}
|
|
and type(policy['schema_version']) is int and policy['schema_version'] == 1
|
|
and policy['profile'] == 'development' and isinstance(policy['expires_at'], str)
|
|
and isinstance(policy['scopes'], list) and 1 <= len(policy['scopes']) <= 64
|
|
and all(isinstance(scope, dict) for scope in policy['scopes']), 'exact legacy Root policy schema')
|
|
expiry = dt.datetime.fromisoformat(policy['expires_at'])
|
|
require(expiry.utcoffset() is not None and expiry <= dt.datetime.now(dt.timezone.utc), 'expired Root policy denies every scope')
|
|
return {'sha256': expected, 'schema_version': 1, 'profile': 'development',
|
|
'expires_at': policy['expires_at'], 'closed': True}
|
|
|
|
def canonical_mounts(mounts):
|
|
# Docker does not promise inspection order. Preserve every object and
|
|
# attribute, reject ambiguous destinations, then compare canonical lists.
|
|
require(isinstance(mounts, list) and all(isinstance(mount, dict)
|
|
and isinstance(mount.get('Destination'), str) and mount['Destination'].startswith('/')
|
|
and isinstance(mount.get('Type'), str) and bool(mount['Type'])
|
|
and isinstance(mount.get('Source'), str) for mount in mounts), 'typed complete Docker mounts')
|
|
require(len({mount['Destination'] for mount in mounts}) == len(mounts), 'unique Docker mount destinations')
|
|
return sorted(mounts, key=lambda mount: (mount['Destination'], mount['Type'], mount['Source']))
|
|
|
|
def container(service):
|
|
values = run(compose + ['ps', '-a', '-q', service]).decode().split()
|
|
require(len(values) == 1, 'one fixed compose service')
|
|
cid = values[0]
|
|
out = {'id': cid}
|
|
for name, fmt in (('image', '.Image'), ('user', '.Config.User'), ('running', '.State.Running'),
|
|
('mounts', '.Mounts'), ('ports', '.HostConfig.PortBindings')):
|
|
out[name] = json.loads(run(['docker', 'inspect', cid, '--format', '{{json ' + fmt + '}}']))
|
|
out['mounts'] = canonical_mounts(out['mounts'])
|
|
if service in ('api', 'gateway'):
|
|
# Filter inside Docker's template: credential env values never
|
|
# enter Python/stdout. No .env, token or private key is opened.
|
|
names = ('RMM_PROFILE', 'RMM_DEV_AUTH', 'RMM_SERVER_ROLE', 'RMM_UI_ORIGIN',
|
|
'RMM_DEV_ROOT_ENABLED', 'RMM_DEV_SYSTEM_ENABLED', 'RMM_DEV_DELIVERY_FAULTS',
|
|
'RMM_DEV_DELIVERY_FAULTS_FILE', 'RMM_DEV_SYSTEM_POLICY_FILE', 'RMM_DEV_ROOT_POLICY_FILE')
|
|
select = ' '.join('(eq $key "' + name + '")' for name in names)
|
|
fmt = '{{range .Config.Env}}{{$key := index (split . "=") 0}}{{if or ' + select + '}}{{println .}}{{end}}{{end}}'
|
|
env = dict(line.split('=', 1) for line in run(['docker', 'inspect', cid, '--format', fmt]).decode().splitlines() if '=' in line)
|
|
out['settings'] = {key: env.get(key) for key in names}
|
|
require(env.get('RMM_PROFILE') == 'development' and env.get('RMM_DEV_AUTH') == 'true'
|
|
and env.get('RMM_SERVER_ROLE') == service and env.get('RMM_UI_ORIGIN') == 'https://localhost:18443'
|
|
and env.get('RMM_DEV_ROOT_ENABLED') == 'true'
|
|
and env.get('RMM_DEV_ROOT_POLICY_FILE') == '/root-policy/policy.json'
|
|
and env.get('RMM_DEV_SYSTEM_ENABLED') is None and env.get('RMM_DEV_DELIVERY_FAULTS') == 'false'
|
|
and env.get('RMM_DEV_DELIVERY_FAULTS_FILE') is None and env.get('RMM_DEV_SYSTEM_POLICY_FILE') is None,
|
|
'unchanged private dev/test role with Root/SYSTEM/fault gates closed')
|
|
mounts = [mount for mount in out['mounts'] if mount['Destination'] == '/root-policy']
|
|
require(len(mounts) == 1 and mounts[0]['Type'] == 'bind' and mounts[0]['RW'] is False
|
|
and mounts[0]['Source'] == str(root / 'devstate/root-policy'), 'exact read-only Root policy mount')
|
|
out['closed_root_policy_sha256'] = closed_root_policy()['sha256']
|
|
require(out['user'] == '10001:10001', 'unprivileged application role')
|
|
port = '18443' if service == 'api' else '18444'
|
|
require(out['ports'] == {'18443/tcp': [{'HostIp': '127.0.0.1', 'HostPort': port}]}, 'loopback-only role')
|
|
else:
|
|
require(not out['ports'], 'PostgreSQL has no published port')
|
|
return out
|
|
|
|
def controls():
|
|
names = ('deploy/Dockerfile', '.dockerignore', 'deploy/server.compose.yaml')
|
|
files = {name: digest(raw(root / name)) for name in names}
|
|
for name in ('rmm-server-ingress.socket', 'rmm-server-ingress.service'):
|
|
files['/etc/systemd/system/' + name] = digest(raw(Path('/etc/systemd/system') / name))
|
|
# Metadata-only identity of secret trees. Private material is never read.
|
|
identities = {}
|
|
for name in ('devstate/.env', 'devstate/pki', 'devstate/api-pki', 'devstate/jobs', 'devstate/root-policy'):
|
|
path = root / name
|
|
paths = [path, *sorted(path.rglob('*'))] if path.is_dir() else [path]
|
|
for item in paths:
|
|
value = item.lstat()
|
|
require(not item.is_symlink() and (stat.S_ISREG(value.st_mode) or stat.S_ISDIR(value.st_mode)), 'preserved state object')
|
|
identities[str(item.relative_to(root))] = [value.st_dev, value.st_ino, value.st_uid, value.st_gid,
|
|
value.st_mode, value.st_size if stat.S_ISREG(value.st_mode) else None,
|
|
value.st_mtime_ns if stat.S_ISREG(value.st_mode) else None]
|
|
ingress = run(['systemctl', 'show', 'rmm-server-ingress.socket', '-p', 'ActiveState', '-p', 'UnitFileState']).decode()
|
|
require('ActiveState=active' in ingress, 'existing public socket ingress stays active')
|
|
return {'files': files, 'protected_metadata': identities, 'ingress': ingress,
|
|
'root_policy': closed_root_policy()}
|
|
|
|
def sql(query):
|
|
return json.loads(run(compose + ['exec', '-T', 'postgres', 'psql', '-X', '-qAt', '-v',
|
|
'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm'], timeout=30, input=query.encode()))
|
|
|
|
def database():
|
|
# One repeatable-read snapshot, only counts and SHA256 values leave PG.
|
|
tables = ('tenants', 'assets', 'operation_plans', 'approval_requests', 'authorizations', 'operations',
|
|
'jobs', 'outbox', 'execution_permits', 'job_receipts', 'audit_events', 'operation_submissions',
|
|
'live_query_requests', 'memberships', 'tenant_limits', 'oidc_principals', 'package_recipes', 'artifacts')
|
|
pairs = []
|
|
for table in tables:
|
|
pairs.append("'" + table + "',(SELECT json_build_object('count',count(*),'sha256',encode(sha256(convert_to("
|
|
"coalesce(string_agg(to_jsonb(t)::text,E'\\n' ORDER BY to_jsonb(t)::text),''),'UTF8')),'hex')) FROM " + table + ' t)')
|
|
# Stable device identity excludes expected changing passive Sync/contact.
|
|
identity = "(SELECT coalesce(json_agg(q ORDER BY tenant_id,agent_id),'[]'::json) FROM (SELECT tenant_id,agent_id,asset_id,enrollment_generation,assignment_version,certificate_sha256,revoked_at FROM agent_installations) q)"
|
|
migrations = "(SELECT json_agg(q ORDER BY version) FROM (SELECT version,description,installed_on,success,encode(checksum,'hex') AS checksum,execution_time FROM _sqlx_migrations) q)"
|
|
role = "(SELECT json_build_object('superuser',rolsuper,'bypassrls',rolbypassrls) FROM pg_roles WHERE rolname='rmm_app')"
|
|
query = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SELECT json_build_object('history',json_build_object(" + ','.join(pairs) + "),'identities'," + identity + ",'migrations'," + migrations + ",'app_role'," + role + ",'unresolved',(SELECT count(*) FROM jobs WHERE state NOT IN ('succeeded','failed','cancelled','expired'))); COMMIT;"
|
|
out = sql(query)
|
|
require(out['unresolved'] == 0 and out['app_role'] == {'superuser': False, 'bypassrls': False}, 'settled history and no RLS bypass')
|
|
require(all(row['success'] for row in out['migrations']), 'successful SQLx ledger')
|
|
return out
|
|
|
|
def witness(running=True):
|
|
current = {'runtime_image': image('rust-discovery-server:local'),
|
|
'controls': controls(), 'containers': {name: container(name) for name in ('postgres', 'api', 'gateway')},
|
|
'index_sha256': digest(raw(root / 'web/dist/index.html')), 'database': database()}
|
|
pg = current['containers']['postgres']
|
|
mounts = [m for m in pg['mounts'] if m['Destination'] == '/var/lib/postgresql/data']
|
|
require(len(mounts) == 1 and mounts[0]['Type'] == 'volume' and pg['running'], 'existing PG data volume')
|
|
value = Path(mounts[0]['Source']).lstat()
|
|
require(stat.S_ISDIR(value.st_mode) and not Path(mounts[0]['Source']).is_symlink(), 'same PG data directory')
|
|
current['pg_volume'] = {'mount': mounts[0], 'device': value.st_dev, 'inode': value.st_ino}
|
|
for name in ('api', 'gateway'):
|
|
require(current['containers'][name]['running'] == running, 'fixed application running state')
|
|
require(current['containers'][name]['image'] == current['runtime_image'], 'both services run runtime tag')
|
|
return current
|
|
|
|
def before_guard(now, expected, stopped=False):
|
|
require(now['runtime_image'] == expected['runtime_image'] == payload['old_image'], 'runtime image CAS')
|
|
for key in ('controls', 'index_sha256', 'database', 'pg_volume'):
|
|
require(now[key] == expected[key], 'preserved ' + key + ' CAS')
|
|
for name in ('postgres', 'api', 'gateway'):
|
|
current = dict(now['containers'][name])
|
|
old = dict(expected['containers'][name])
|
|
if stopped and name != 'postgres':
|
|
current.pop('running'); old.pop('running')
|
|
require(current == old, 'fixed prior service/container CAS')
|
|
|
|
def migration_guard(record, after=False):
|
|
expected = record['migration_checksums']
|
|
require([row['version'] for row in record['baseline']['database']['migrations']] == list(range(1, 14)), 'exact prior migrations 1..13')
|
|
for row in record['baseline']['database']['migrations']:
|
|
require(row['checksum'] == expected[str(row['version'])], 'frozen old migration checksum')
|
|
if after:
|
|
now = database()
|
|
require(now['migrations'][:13] == record['baseline']['database']['migrations']
|
|
and [row['version'] for row in now['migrations']] == list(range(1, 15))
|
|
and now['migrations'][13]['checksum'] == expected['14'], 'only additive frozen migration14')
|
|
for key in ('history', 'identities', 'app_role', 'unresolved'):
|
|
require(now[key] == record['baseline']['database'][key], 'original database rows unchanged')
|
|
check = sql("BEGIN READ ONLY; SELECT json_build_object('count',(SELECT count(*) FROM windows_msi_recipes),"
|
|
"'rls',(SELECT relrowsecurity AND relforcerowsecurity FROM pg_class WHERE oid='windows_msi_recipes'::regclass),"
|
|
"'select',has_table_privilege('rmm_app','windows_msi_recipes','SELECT'),"
|
|
"'insert',has_table_privilege('rmm_app','windows_msi_recipes','INSERT'),"
|
|
"'update',has_table_privilege('rmm_app','windows_msi_recipes','UPDATE'),"
|
|
"'delete',has_table_privilege('rmm_app','windows_msi_recipes','DELETE'),"
|
|
"'trigger',EXISTS(SELECT 1 FROM pg_trigger WHERE tgrelid='windows_msi_recipes'::regclass AND tgname='immutable_content' AND tgenabled='O')); COMMIT;")
|
|
require(check == {'count': 0, 'rls': True, 'select': True, 'insert': True, 'update': False, 'delete': False, 'trigger': True}, 'empty immutable MSI tenant catalog with FORCE RLS')
|
|
return now
|
|
|
|
def check_source(record):
|
|
require(all(digest(raw(folder / 'context' / name)) == value for name, value in record['source_files'].items()), 'frozen build context unchanged')
|
|
|
|
def https(port, path, host='localhost'):
|
|
require((port, host) in ((18443, 'localhost'), (18444, 'localhost'),
|
|
(443, 'discovery.ease-technologies.de')), 'fixed role TLS endpoint and name')
|
|
ca = root / ('devstate/api-pki/ca.pem' if port == 18443 else 'devstate/pki/ca.pem')
|
|
class HTTPS(http.client.HTTPSConnection):
|
|
def connect(self):
|
|
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', port), timeout=5), server_hostname=self.host)
|
|
client = HTTPS(host, port, timeout=5, context=ssl.create_default_context(cafile=str(ca)))
|
|
try:
|
|
client.request('GET', path)
|
|
response = client.getresponse()
|
|
body = response.read(2 * 1024 * 1024 + 1)
|
|
require(len(body) <= 2 * 1024 * 1024, 'bounded HTTPS body')
|
|
return {'status': response.status, 'sha256': digest(body), 'body': body}
|
|
finally:
|
|
client.close()
|
|
|
|
def health():
|
|
for port in (18443, 18444):
|
|
deadline = time.monotonic() + 60
|
|
while True:
|
|
try:
|
|
result = https(port, '/healthz')
|
|
require(result['status'] == 200 and result['body'] == b'ok', 'healthy exact server role')
|
|
break
|
|
except Exception:
|
|
require(time.monotonic() < deadline, 'TLS role health deadline')
|
|
time.sleep(1)
|
|
auth = https(18443, '/auth/config')
|
|
require(auth['status'] == 200 and json.loads(auth['body']) ==
|
|
{'oidc_configured': False, 'oidc_login_url': None, 'dev_login_enabled': True}, 'honest private dev/test authentication')
|
|
for path in ('/', '/auth/config', '/auth/session', '/auth/dev/login', '/mcp', '/api/v1/tenants'):
|
|
require(https(443, path, 'discovery.ease-technologies.de')['status'] == 404, 'public gateway has no UI/auth/MCP/admin API')
|
|
require(https(443, '/api/v1/job-signing-key', 'discovery.ease-technologies.de')['status'] == 401, 'public signing key requires authenticated agent')
|
|
|
|
def replace_public(path, data):
|
|
require(path.parent.is_dir() and not path.is_symlink(), 'existing served directory, no symlink')
|
|
next_path = path.with_name(path.name + '.script-console-' + ident)
|
|
once(next_path, data, 0o644)
|
|
os.replace(next_path, path)
|
|
syncdir(path.parent)
|
|
|
|
def after_guard(built, index):
|
|
now = witness()
|
|
require(now['runtime_image'] == built['image_id'], 'exact candidate remains runtime')
|
|
for key in ('controls', 'pg_volume'):
|
|
require(now[key] == record['baseline'][key], 'preserved post-deploy ' + key)
|
|
require(now['index_sha256'] == index, 'exact post-deploy index')
|
|
require(now['containers']['postgres'] == record['baseline']['containers']['postgres'], 'same PG container')
|
|
for name in ('api', 'gateway'):
|
|
before = dict(record['baseline']['containers'][name])
|
|
after = dict(now['containers'][name])
|
|
for key in ('id', 'image'):
|
|
before.pop(key); after.pop(key)
|
|
require(before == after, 'same application role, mounts, ports and closed gates')
|
|
migration_guard(record, after=True)
|
|
|
|
# inspect and status never mkdir, upload, launch, migrate, retag or publish.
|
|
if action == 'inspect':
|
|
return witness()
|
|
if action == 'build':
|
|
record = payload['record']
|
|
require(record['id'] == ident and record['old_image'] == payload['old_image'], 'fixed prepared case')
|
|
before_guard(witness(), record['baseline'])
|
|
migration_guard(record)
|
|
require(record['baseline']['controls']['files']['deploy/Dockerfile'] == record['source_files']['deploy/Dockerfile']
|
|
and record['baseline']['controls']['files']['.dockerignore'] == record['source_files']['.dockerignore']
|
|
and record['baseline']['controls']['files']['deploy/server.compose.yaml'] == record['source_files']['deploy/server.compose.yaml'], 'preserving unchanged deploy controls')
|
|
# Create once before the first write; an incomplete case is status-only.
|
|
owned(root, True); owned(root / 'devstate', True)
|
|
folder.mkdir(mode=0o700); syncdir(folder.parent)
|
|
once(folder / 'intent.json', enc(record))
|
|
context = folder / 'context'; context.mkdir(mode=0o700)
|
|
archive_raw = base64.b64decode(payload['source_archive'], validate=True)
|
|
require(len(archive_raw) <= 20 * 1024 * 1024 and digest(archive_raw) == record['source_archive_sha256'], 'source selection archive')
|
|
found = set()
|
|
with tarfile.open(fileobj=io.BytesIO(archive_raw), mode='r:') as archive:
|
|
for member in archive.getmembers():
|
|
name = member.name
|
|
require(member.isfile() and name in record['source_files'] and name not in found
|
|
and not Path(name).is_absolute() and '..' not in Path(name).parts
|
|
and member.size <= 2 * 1024 * 1024, 'fixed complete source selection')
|
|
data = archive.extractfile(member).read()
|
|
require(digest(data) == record['source_files'][name], 'source member hash')
|
|
dest = context / name; dest.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
once(dest, data); found.add(name)
|
|
require(found == set(record['source_files']), 'complete frozen source selection')
|
|
web = base64.b64decode(payload['web_archive'], validate=True)
|
|
require(digest(web) == record['web_archive_sha256'] and len(web) < 2 * 1024 * 1024, 'qualified Web49 archive')
|
|
once(folder / 'web.tar', web)
|
|
worker = payload['worker_source'].encode()
|
|
require(digest(worker) == record['worker_source_sha256'], 'fixed worker source')
|
|
once(folder / 'worker.py', worker)
|
|
once(folder / 'build-started.json', enc({'id': ident, 'intent_sha256': digest(enc(record))}))
|
|
run(['systemd-run', '--unit=rmm-script-build-' + ident, '--property=UMask=0077',
|
|
'--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'build-worker'], timeout=15)
|
|
return {'id': ident, 'build_launched': True, 'deployed': False, 'lost_reply_action': 'status only'}
|
|
|
|
if action == 'status':
|
|
if not folder.exists():
|
|
require(not folder.is_symlink(), 'no case symlink')
|
|
return {'id': ident, 'case_present': False, 'read_only': True, 'retry_permitted': False}
|
|
owned(folder, True)
|
|
receipts = {}
|
|
for path in sorted(folder.glob('*.json')):
|
|
if path.name != 'intent.json':
|
|
try:
|
|
receipts[path.name] = load(path)
|
|
except Exception as error:
|
|
receipts[path.name] = {'read_error_class': type(error).__name__}
|
|
result = {'id': ident, 'case_present': True, 'receipts': receipts, 'read_only': True, 'retry_permitted': False}
|
|
# Readback must survive an interrupted upload, lost launch reply, failed
|
|
# migration or stopped roles. Report partial evidence; never heal it.
|
|
checks = {'runtime_image': lambda: image('rust-discovery-server:local'), 'database': database,
|
|
'containers': lambda: {name: container(name) for name in ('postgres', 'api', 'gateway')},
|
|
'controls': controls, 'index_sha256': lambda: digest(raw(root / 'web/dist/index.html'))}
|
|
if (folder / 'intent.json').exists():
|
|
checks['intent_sha256'] = lambda: digest(raw(folder / 'intent.json', 8 * 1024 * 1024))
|
|
checks['source_unchanged'] = lambda: all((folder / 'context' / name).is_file()
|
|
and digest(raw(folder / 'context' / name)) == value
|
|
for name, value in load(folder / 'intent.json')['source_files'].items())
|
|
def served_web():
|
|
values = {}
|
|
for name in load(folder / 'intent.json')['web_files']:
|
|
got = https(18443, '/' + name if name != 'index.html' else '/')
|
|
values[name] = {'status': got['status'], 'sha256': got['sha256']}
|
|
return values
|
|
checks['private_TLS_web'] = served_web
|
|
for name, check in checks.items():
|
|
try:
|
|
result[name] = check()
|
|
except Exception as error:
|
|
result[name] = {'read_error_class': type(error).__name__}
|
|
for name in ('build', 'deploy'):
|
|
path = folder / (name + '.log')
|
|
result[name + '_log_bytes'] = path.stat().st_size if path.exists() else None
|
|
return result
|
|
|
|
owned(folder, True)
|
|
record = load(folder / 'intent.json')
|
|
require(record['id'] == ident and record['old_image'] == payload['old_image'], 'same case intent')
|
|
require(digest(raw(folder / 'worker.py')) == record['worker_source_sha256'], 'retained worker unchanged')
|
|
def deploy_guard():
|
|
check_source(record)
|
|
before_guard(witness(), record['baseline'])
|
|
migration_guard(record)
|
|
built = load(folder / 'build-result.json')
|
|
require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'exact successful candidate image')
|
|
require(not (folder / 'deploy-started.json').exists(), 'create-once deployment; status only after any launch')
|
|
return built
|
|
|
|
if action == 'deploy':
|
|
built = deploy_guard()
|
|
once(folder / 'deploy-started.json', enc({'id': ident, 'image_id': built['image_id'], 'intent_sha256': digest(enc(record))}))
|
|
run(['systemd-run', '--unit=rmm-script-deploy-' + ident, '--property=UMask=0077',
|
|
'--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'deploy-worker'], timeout=15)
|
|
return {'id': ident, 'deploy_launched': True, 'completed': False, 'lost_reply_action': 'status only'}
|
|
|
|
# The retained worker, never caller-supplied shell code, performs each phase
|
|
# once. A host-wide lock serializes this runner's build/deployment workers.
|
|
require(action in ('build-worker', 'deploy-worker'), 'retained worker action')
|
|
lock = os.open(root / 'devstate/.script-console-upgrade-v1.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
phase = 'build' if action == 'build-worker' else 'deploy'
|
|
try:
|
|
once(folder / (phase + '-worker-started.json'), enc({'id': ident}))
|
|
if phase == 'build':
|
|
before_guard(witness(), record['baseline'])
|
|
check_source(record)
|
|
require(not run(['docker', 'image', 'ls', '-q', record['candidate_tag']]).strip(), 'new candidate tag only')
|
|
with (folder / 'build.log').open('xb') as log:
|
|
result = subprocess.run(['docker', 'build', '--platform=linux/arm64', '--file', str(folder / 'context/deploy/Dockerfile'),
|
|
'--tag', record['candidate_tag'], str(folder / 'context')], stdout=log, stderr=subprocess.STDOUT, timeout=3600)
|
|
log.flush(); os.fsync(log.fileno())
|
|
require(result.returncode == 0, 'native ARM64 image build failed')
|
|
check_source(record)
|
|
before_guard(witness(), record['baseline'])
|
|
outcome = {'passed': True, 'image_id': image(record['candidate_tag']), 'source_files_equal': True,
|
|
'runtime_unchanged': True, 'migration_executed': False, 'jobs_or_approvals_created': False}
|
|
else:
|
|
check_source(record)
|
|
built = load(folder / 'build-result.json')
|
|
require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'fixed built image')
|
|
before_guard(witness(), record['baseline'])
|
|
migration_guard(record)
|
|
# Intent already durable. Stop both writers before checking/backup;
|
|
# failures deliberately leave an incomplete case for operator readback.
|
|
run(compose + ['stop', '--timeout', '20', 'api', 'gateway'], timeout=50)
|
|
once(folder / '01-services-stopped.json', enc({'id': ident}))
|
|
before_guard(witness(running=False), record['baseline'], stopped=True)
|
|
active = sql("BEGIN READ ONLY; SELECT json_build_object('other_sessions',count(*)) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND backend_type='client backend' AND application_name<>'pg_isready'; COMMIT;")
|
|
require(active['other_sessions'] == 0, 'no other database writer during upgrade')
|
|
with (folder / 'postgres-before.dump').open('xb') as dump:
|
|
result = subprocess.run(compose + ['exec', '-T', 'postgres', 'pg_dump', '-U', 'postgres', '-d', 'rmm', '--format=custom'],
|
|
stdout=dump, stderr=subprocess.PIPE, timeout=120)
|
|
dump.flush(); os.fsync(dump.fileno())
|
|
require(result.returncode == 0, 'private pre-migration database backup')
|
|
once(folder / 'index-before.html', raw(root / 'web/dist/index.html'))
|
|
once(folder / '02-backup-complete.json', enc({'id': ident, 'backup_private': True}))
|
|
# Parse/check the entire UI archive before publishing any asset.
|
|
web_raw = raw(folder / 'web.tar')
|
|
require(digest(web_raw) == record['web_archive_sha256'], 'unchanged UI archive')
|
|
web = {}
|
|
with tarfile.open(fileobj=io.BytesIO(web_raw), mode='r:') as archive:
|
|
for member in archive.getmembers():
|
|
require(member.isfile() and member.name in record['web_files'] and member.name not in web
|
|
and member.size <= 2 * 1024 * 1024, 'exact qualified UI members')
|
|
data = archive.extractfile(member).read()
|
|
require(digest(data) == record['web_files'][member.name], 'qualified UI bytes')
|
|
web[member.name] = data
|
|
require(set(web) == set(record['web_files']), 'complete qualified UI')
|
|
dist = root / 'web/dist'; owned(dist, True); owned(dist / 'assets', True)
|
|
for name, data in web.items():
|
|
if name == 'index.html':
|
|
continue
|
|
dest = dist / name
|
|
if dest.exists():
|
|
require(digest(raw(dest)) == record['web_files'][name], 'immutable asset filename collision')
|
|
else:
|
|
once(dest, data, 0o644)
|
|
once(folder / '03-assets-published.json', enc({'id': ident, 'index_unchanged': digest(raw(dist / 'index.html')) == record['baseline']['index_sha256']}))
|
|
require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'old index CAS before migration')
|
|
require(image('rust-discovery-server:local') == record['old_image'], 'runtime tag CAS before transition')
|
|
run(['docker', 'tag', built['image_id'], 'rust-discovery-server:local'])
|
|
once(folder / '04-runtime-tagged.json', enc({'id': ident, 'image_id': built['image_id']}))
|
|
# Named migration container remains inspectable even on failure/lost
|
|
# reply. Never remove it or issue a second migration automatically.
|
|
with (folder / 'deploy.log').open('xb') as log:
|
|
result = subprocess.run(compose + ['run', '--name', 'rmm-script-migrate-' + ident,
|
|
'--no-deps', 'migrate'], stdout=log, stderr=subprocess.STDOUT, timeout=120)
|
|
log.flush(); os.fsync(log.fileno())
|
|
require(result.returncode == 0, 'migration incomplete; inspect retained named container')
|
|
migration_guard(record, after=True)
|
|
once(folder / '05-migration14-verified.json', enc({'id': ident, 'original_ledger_and_history_equal': True}))
|
|
run(compose + ['up', '-d', '--no-deps', '--no-build', 'api', 'gateway'], timeout=60)
|
|
health()
|
|
after_guard(built, record['baseline']['index_sha256'])
|
|
once(folder / '06-backend-ready.json', enc({'id': ident, 'TLS_healthy': True, 'public_admin_routes_closed': True}))
|
|
# Directory inode and all old hashed assets survive. Only now switch
|
|
# the entry document, atomically and against the exact old index.
|
|
require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'final old index CAS')
|
|
replace_public(dist / 'index.html', web['index.html'])
|
|
for name, expected in record['web_files'].items():
|
|
served = https(18443, '/' + name if name != 'index.html' else '/')
|
|
require(served['status'] == 200 and served['sha256'] == expected, 'actual private TLS UI byte hash')
|
|
after_guard(built, record['web_files']['index.html'])
|
|
once(folder / '07-index-published.json', enc({'id': ident, 'web_files': record['web_files']}))
|
|
outcome = {'passed': True, 'image_id': built['image_id'], 'migration14': True, 'web_files': record['web_files'],
|
|
'prior_history_equal': True, 'jobs_or_approvals_created': False, 'root_system_fault_gates_closed': True,
|
|
'production_oidc_qualified': False, 'native_client_acceptance': False}
|
|
once(folder / (phase + '-result.json'), enc(outcome))
|
|
return outcome
|
|
except Exception as error:
|
|
# No raw exception/command/DB output can escape to the caller or log.
|
|
failure = {'passed': False, 'error_class': type(error).__name__, 'retry_permitted': False,
|
|
'frames': [{'function': frame.name, 'line': frame.lineno} for frame in traceback.extract_tb(error.__traceback__)]}
|
|
once(folder / (phase + '-failure.json'), enc(failure))
|
|
raise RuntimeError('case incomplete; status/readback only') from None
|
|
finally:
|
|
os.close(lock)
|
|
|
|
|
|
def archive(files):
|
|
stream = io.BytesIO()
|
|
with tarfile.open(fileobj=stream, mode='w:') as tar:
|
|
for name, raw in sorted(files.items()):
|
|
item = tarfile.TarInfo(name)
|
|
item.size = len(raw)
|
|
item.mode = 0o644
|
|
tar.addfile(item, io.BytesIO(raw))
|
|
return stream.getvalue()
|
|
|
|
|
|
def worker_source(ident):
|
|
prefix = 'import os,json,subprocess,base64,hashlib,uuid,sys\nfrom pathlib import Path\n'
|
|
return (prefix + inspect.getsource(bootstrap.require) + '\n' + inspect.getsource(remote) +
|
|
'\nif __name__ == "__main__":\n remote(dict(action=sys.argv[1],id=' + repr(ident) + ',old_image=' + repr(OLD_IMAGE) + '))\n').encode()
|
|
|
|
|
|
def check_support():
|
|
require(all(sha((ROOT / name).read_bytes()) == expected for name, expected in SUPPORT.items()), 'pinned SSH support sources')
|
|
|
|
|
|
def prepare(baseline_path):
|
|
require(sha(FROZEN.read_bytes()) == FROZEN_SHA, 'frozen158 qualification manifest')
|
|
frozen = json.loads(FROZEN.read_bytes())
|
|
require(frozen['source_before_after_equal'] and len(frozen['source_files']) == 158
|
|
and all(check['exit_code'] == 0 for check in frozen['checks'].values()), 'passing exact qualification')
|
|
source_raw = (FROZEN.parent / 'reviewed-sources.tar.gz').read_bytes()
|
|
require(sha(source_raw) == frozen['source_archive_sha256'], 'frozen source archive')
|
|
sources = {}
|
|
with tarfile.open(fileobj=io.BytesIO(source_raw), mode='r:gz') as tar:
|
|
for item in tar.getmembers():
|
|
require(item.isfile() and item.name in frozen['source_files'] and item.name not in sources
|
|
and item.size <= 2 * 1024 * 1024 and not Path(item.name).is_absolute()
|
|
and '..' not in Path(item.name).parts, 'frozen source selection')
|
|
data = tar.extractfile(item).read()
|
|
require(sha(data) == frozen['source_files'][item.name], 'frozen member hash')
|
|
sources[item.name] = data
|
|
require(set(sources) == set(frozen['source_files']), 'all158 frozen source inputs retained')
|
|
for name, expected in EXTRA.items():
|
|
path = ROOT / name
|
|
require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'separate deployment source pin')
|
|
sources[name] = path.read_bytes()
|
|
web = {}
|
|
for name, expected in WEB.items():
|
|
path = ROOT / 'web/dist' / name
|
|
require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'qualified Web49 byte pin')
|
|
web[name] = path.read_bytes()
|
|
# A complete context is manifest-selected, not copied from a live checkout.
|
|
for forbidden in ('credentials/', 'devstate/', '.git/', 'target/', 'web/node_modules/'):
|
|
require(not any(name.startswith(forbidden) or '/' + forbidden in name for name in sources), 'private/cache path excluded')
|
|
require(all(name in sources for name in ('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml',
|
|
'crates/oidc-auth/sql/001_oidc_login_states.sql', 'tools/apt-transaction/apt_transaction.py',
|
|
'tools/apt-transaction/hook', 'deploy/linux/rmm-apt-helper.service', 'migrations/0014_windows_msi_catalog.sql')), 'Docker COPY/include closure')
|
|
baseline_raw = baseline_path.read_bytes()
|
|
baseline = json.loads(baseline_raw)
|
|
require(baseline['runtime_image'] == OLD_IMAGE and baseline['database']['unresolved'] == 0
|
|
and [row['version'] for row in baseline['database']['migrations']] == list(range(1, 14)), 'fresh intended prior runtime/schema')
|
|
ident = str(uuid.uuid4())
|
|
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
folder = STATE / ident
|
|
folder.mkdir(mode=0o700)
|
|
data = archive(sources); web_data = archive(web); worker = worker_source(ident)
|
|
checksums = {str(int(Path(name).name.split('_', 1)[0])): hashlib.sha384(value).hexdigest()
|
|
for name, value in sources.items() if re.fullmatch(r'migrations/\d{4}_[a-z0-9_]+\.sql', name)}
|
|
require(set(checksums) == {str(n) for n in range(1, 15)}, 'exact migration1..14 closure')
|
|
record = {'schema_version': 1, 'type': 'rmm.script-console.cloud-upgrade.v2', 'id': ident,
|
|
'frozen_manifest_sha256': FROZEN_SHA, 'old_image': OLD_IMAGE, 'baseline': baseline,
|
|
'baseline_sha256': sha(baseline_raw), 'source_files': {name: sha(value) for name, value in sources.items()},
|
|
'source_archive_sha256': sha(data), 'web_files': WEB, 'web_archive_sha256': sha(web_data),
|
|
'worker_source_sha256': sha(worker), 'runner_sha256': sha(Path(__file__).read_bytes()),
|
|
'support_sha256': SUPPORT, 'migration_checksums': checksums,
|
|
'candidate_tag': 'rust-discovery-server:script-v2-' + ident}
|
|
record['supersedes_failed_case'] = PREVIOUS_FAILED_CASE
|
|
for row in baseline['database']['migrations']:
|
|
require(row['success'] and row['checksum'] == checksums[str(row['version'])], 'unchanged SQLx1..13 bytes')
|
|
for name, raw in (('source.tar', data), ('web.tar', web_data), ('worker.py', worker),
|
|
('baseline.json', baseline_raw), ('manifest.json', encoded(record))):
|
|
create(folder / name, raw)
|
|
return {'id': ident, 'manifest': str(folder / 'manifest.json'), 'manifest_sha256': sha(encoded(record)),
|
|
'source_count': len(sources), 'remote_operations': False, 'build_started': False}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('inspect', 'prepare', 'build', 'deploy', 'status'))
|
|
parser.add_argument('--id')
|
|
parser.add_argument('--baseline', type=Path)
|
|
parser.add_argument('--access-root', type=Path, default=ROOT,
|
|
help='existing main checkout containing protected server.conf; SSH only')
|
|
parser.add_argument('--execute-cloud-build', action='store_true')
|
|
parser.add_argument('--execute-cloud-deploy', action='store_true')
|
|
args = parser.parse_args()
|
|
os.umask(0o077)
|
|
check_support()
|
|
if args.action == 'prepare':
|
|
require(args.baseline is not None, 'explicit saved remote baseline required')
|
|
print(json.dumps(prepare(args.baseline)))
|
|
return
|
|
# Keep the reviewed worktree's source inputs, but reuse the existing main
|
|
# checkout's protected SSH transport configuration when explicitly selected.
|
|
bootstrap.ROOT = args.access_root.resolve()
|
|
bootstrap.access.ROOT = bootstrap.ROOT
|
|
config = bootstrap.access.load_config(bootstrap.ROOT / 'credentials/server.conf')
|
|
require(config['address'] in ('88.99.80.202', 'discovery.ease-technologies.de')
|
|
and config['username'] == 'root' and config.get('port', '22') == '22', 'fixed existing cloud server bootstrap target')
|
|
if args.action == 'inspect':
|
|
result = bootstrap.rpc('server.conf', remote, {'action': 'inspect', 'id': str(uuid.uuid4()), 'old_image': OLD_IMAGE})
|
|
require(result['runtime_image'] == OLD_IMAGE, 'known actual prior server image')
|
|
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
path = STATE / ('baseline-' + str(uuid.uuid4()) + '.json')
|
|
create(path, encoded(result))
|
|
print(json.dumps({'baseline': str(path), 'sha256': sha(encoded(result)), 'read_only_remote': True}))
|
|
return
|
|
ident = str(uuid.UUID(args.id))
|
|
require(ident == args.id, 'canonical case UUID')
|
|
folder = STATE / ident
|
|
manifest = json.loads((folder / 'manifest.json').read_bytes())
|
|
require(manifest['id'] == ident and manifest['runner_sha256'] == sha(Path(__file__).read_bytes())
|
|
and manifest['support_sha256'] == SUPPORT and manifest['worker_source_sha256'] == sha(worker_source(ident)), 'retained exact runner/support/worker')
|
|
payload = {'action': args.action, 'id': ident, 'old_image': OLD_IMAGE}
|
|
if args.action in ('build', 'deploy'):
|
|
require(args.execute_cloud_build if args.action == 'build' else args.execute_cloud_deploy, 'explicit bounded cloud action required')
|
|
# Local once intent BEFORE SSH: uncertain/lost replies cannot launch twice.
|
|
create(folder / (args.action + '-requested.json'), encoded({'id': ident, 'manifest_sha256': sha(encoded(manifest))}))
|
|
if args.action == 'build':
|
|
payload.update(record=manifest, source_archive=base64.b64encode((folder / 'source.tar').read_bytes()).decode(),
|
|
web_archive=base64.b64encode((folder / 'web.tar').read_bytes()).decode(), worker_source=(folder / 'worker.py').read_text())
|
|
result = bootstrap.rpc('server.conf', remote, payload)
|
|
create(folder / (args.action + '-reply-' + str(uuid.uuid4()) + '.json'), encoded(result))
|
|
print(json.dumps(result))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
print('SCRIPT_CONSOLE_UPGRADE_INCOMPLETE:' + type(error).__name__ + '; preserve case, status only after any request')
|
|
raise SystemExit(2) from None
|