Preserve script bootstrap and cloud build attempts with exact evidence
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-14 17:42:44 +01:00
parent ff78b6a498
commit 95296bb53b
10 changed files with 2557 additions and 0 deletions
+692
View File
@@ -0,0 +1,692 @@
#!/usr/bin/env python3
"""One frozen Script1.20/MSI1.19 server upgrade; no jobs, approvals or policies.
prepare is offline. inspect/status are remote read-only. build/deploy each launch
one durable, create-once case worker. Lost replies permit status only, not retry.
The notebook never runs an RMM server/client. Existing deployment tools remain
unchanged. See docs/script-console-cloud-upgrade-v1.md before actual use.
"""
import argparse
import base64
import hashlib
import inspect
import io
import json
import os
from pathlib import Path
import re
import tarfile
import uuid
import remote_client_bootstrap as bootstrap
ROOT = Path(__file__).resolve().parents[1]
STATE = ROOT / 'devstate/script-console-cloud-upgrade-v1'
FROZEN = ROOT / 'devstate/script-console-qualification/run-29461892-343f-42f4-b469-88703535425e/manifest.json'
FROZEN_SHA = 'ddd1a78d1cd289b02b1f98572f9652171ec00be84e01ce099fd722d689bcaacd'
OLD_IMAGE = 'sha256:25697f948325a11f755893e16ec5be2da138c3333c8e60f75a18f973d6ac26e2'
EXTRA = {
'deploy/Dockerfile': 'd41b4ec902c11a9e25c22b1c1d3d60d15afc207f2598ecf559cf3bdae4f8e782',
'.dockerignore': 'fdb5cba5967249f5dac35d45c5edf0be2a4e1594c94d04a2ed416d89821ace75',
'deploy/server.compose.yaml': 'd2044b0ee6d584f360990635bec149511d5a0623ec562976541900d751998818',
}
WEB = {
'index.html': '95d6f9058cc713c6af36fbd22ccc60678f081765a853b53d19ac6f0d219804f8',
'assets/index-Be1zm1OZ.css': '916e8870d4e9a9553de452b3ff6c71671bb6a7a22355de0aff432db516a9f3e7',
'assets/index-CTgvl2jr.js': 'f611662042eb4cc44f25c7d5ddc0d2d0562e32be267d3d5fb1db31b0d310f879',
}
SUPPORT = {
'deploy/remote_client_bootstrap.py': '785c43a51dfe3f93e39798dbd7611c353752ae2fc64b22e41ba168e221620d30',
'deploy/vm_access_check.py': '5c8107ea9151a9f498b2530cdc61c1a7a6d745aff853a9b05710f3395838c0dd',
}
require = bootstrap.require
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def encoded(value):
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def create(path, raw):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw)
stream.flush()
os.fsync(stream.fileno())
fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(fd)
finally:
os.close(fd)
def remote(payload):
"""Self-contained function transported by the existing strict-hostkey RPC."""
import datetime as dt
import fcntl
import http.client
import io
import platform
import socket
import ssl
import stat
import subprocess
import tarfile
import time
import traceback
os.umask(0o077)
root = Path('/opt/rust-discovery')
action = payload['action']
require(os.geteuid() == 0 and platform.machine() == 'aarch64', 'fixed root ARM64 server')
require(action in ('inspect', 'build', 'build-worker', 'deploy', 'deploy-worker', 'status'), 'fixed upgrade action')
ident = str(uuid.UUID(payload['id']))
require(ident == payload['id'], 'canonical case UUID')
folder = root / 'devstate' / ('script-console-upgrade-v1-' + ident)
compose = ['docker', 'compose', '--env-file', str(root / 'devstate/.env'),
'-f', str(root / 'deploy/server.compose.yaml')]
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def enc(value):
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def run(args, timeout=20, **kwargs):
result = subprocess.run(args, capture_output=True, timeout=timeout, **kwargs)
require(result.returncode == 0, 'fixed server command incomplete; inspect private case')
return result.stdout
def owned(path, directory=False):
value = path.lstat()
require(value.st_uid == 0 and not value.st_mode & 0o022 and not path.is_symlink()
and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'root-owned fixed path')
return value
def raw(path, limit=2 * 1024 * 1024):
value = owned(path)
require(value.st_size <= limit and value.st_nlink == 1, 'bounded private regular file')
return path.read_bytes()
def syncdir(path):
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(fd)
finally:
os.close(fd)
def once(path, data, mode=0o600):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(data)
stream.flush()
os.fchmod(stream.fileno(), mode)
os.fsync(stream.fileno())
syncdir(path.parent)
def load(path):
return json.loads(raw(path, 8 * 1024 * 1024))
def image(tag):
# Separate fields avoid ever retrieving environment variables/secrets.
value = run(['docker', 'image', 'inspect', tag, '--format', '{{.Id}} {{.Architecture}} {{.Os}}']).decode().split()
require(len(value) == 3 and value[1:] == ['arm64', 'linux'] and value[0].startswith('sha256:'), 'native ARM64 image')
return value[0]
def closed_root_policy():
# Existing MVP state: flag=true, but this exact historical policy is
# expired. Neither the flag nor the policy is changed by deployment.
directory = root / 'devstate/root-policy'
path = directory / 'policy.json'
parent = directory.lstat(); value = path.lstat()
require(not directory.is_symlink() and stat.S_ISDIR(parent.st_mode)
and parent.st_uid == 10001 and not parent.st_mode & 0o022
and not path.is_symlink() and stat.S_ISREG(value.st_mode)
and value.st_uid == 10001 and stat.S_IMODE(value.st_mode) == 0o600
and value.st_nlink == 1 and 0 < value.st_size <= 65536, 'fixed protected closed Root policy path')
data = path.read_bytes()
expected = 'cf15cbb68b590475df6def28e7dcffb7d88e0ca2d2555a06174fb654eeff05f4'
require(digest(data) == expected, 'exact historical closed Root policy hash')
policy = json.loads(data)
require(isinstance(policy, dict) and set(policy) == {'schema_version', 'profile', 'expires_at', 'scopes'}
and type(policy['schema_version']) is int and policy['schema_version'] == 1
and policy['profile'] == 'development' and isinstance(policy['expires_at'], str)
and isinstance(policy['scopes'], list) and 1 <= len(policy['scopes']) <= 64
and all(isinstance(scope, dict) for scope in policy['scopes']), 'exact legacy Root policy schema')
expiry = dt.datetime.fromisoformat(policy['expires_at'])
require(expiry.utcoffset() is not None and expiry <= dt.datetime.now(dt.timezone.utc), 'expired Root policy denies every scope')
return {'sha256': expected, 'schema_version': 1, 'profile': 'development',
'expires_at': policy['expires_at'], 'closed': True}
def container(service):
values = run(compose + ['ps', '-a', '-q', service]).decode().split()
require(len(values) == 1, 'one fixed compose service')
cid = values[0]
out = {'id': cid}
for name, fmt in (('image', '.Image'), ('user', '.Config.User'), ('running', '.State.Running'),
('mounts', '.Mounts'), ('ports', '.HostConfig.PortBindings')):
out[name] = json.loads(run(['docker', 'inspect', cid, '--format', '{{json ' + fmt + '}}']))
if service in ('api', 'gateway'):
# Filter inside Docker's template: credential env values never
# enter Python/stdout. No .env, token or private key is opened.
names = ('RMM_PROFILE', 'RMM_DEV_AUTH', 'RMM_SERVER_ROLE', 'RMM_UI_ORIGIN',
'RMM_DEV_ROOT_ENABLED', 'RMM_DEV_SYSTEM_ENABLED', 'RMM_DEV_DELIVERY_FAULTS',
'RMM_DEV_DELIVERY_FAULTS_FILE', 'RMM_DEV_SYSTEM_POLICY_FILE', 'RMM_DEV_ROOT_POLICY_FILE')
select = ' '.join('(eq $key "' + name + '")' for name in names)
fmt = '{{range .Config.Env}}{{$key := index (split . "=") 0}}{{if or ' + select + '}}{{println .}}{{end}}{{end}}'
env = dict(line.split('=', 1) for line in run(['docker', 'inspect', cid, '--format', fmt]).decode().splitlines() if '=' in line)
out['settings'] = {key: env.get(key) for key in names}
require(env.get('RMM_PROFILE') == 'development' and env.get('RMM_DEV_AUTH') == 'true'
and env.get('RMM_SERVER_ROLE') == service and env.get('RMM_UI_ORIGIN') == 'https://localhost:18443'
and env.get('RMM_DEV_ROOT_ENABLED') == 'true'
and env.get('RMM_DEV_ROOT_POLICY_FILE') == '/root-policy/policy.json'
and env.get('RMM_DEV_SYSTEM_ENABLED') is None and env.get('RMM_DEV_DELIVERY_FAULTS') == 'false'
and env.get('RMM_DEV_DELIVERY_FAULTS_FILE') is None and env.get('RMM_DEV_SYSTEM_POLICY_FILE') is None,
'unchanged private dev/test role with Root/SYSTEM/fault gates closed')
mounts = [mount for mount in out['mounts'] if mount['Destination'] == '/root-policy']
require(len(mounts) == 1 and mounts[0]['Type'] == 'bind' and mounts[0]['RW'] is False
and mounts[0]['Source'] == str(root / 'devstate/root-policy'), 'exact read-only Root policy mount')
out['closed_root_policy_sha256'] = closed_root_policy()['sha256']
require(out['user'] == '10001:10001', 'unprivileged application role')
port = '18443' if service == 'api' else '18444'
require(out['ports'] == {'18443/tcp': [{'HostIp': '127.0.0.1', 'HostPort': port}]}, 'loopback-only role')
else:
require(not out['ports'], 'PostgreSQL has no published port')
return out
def controls():
names = ('deploy/Dockerfile', '.dockerignore', 'deploy/server.compose.yaml')
files = {name: digest(raw(root / name)) for name in names}
for name in ('rmm-server-ingress.socket', 'rmm-server-ingress.service'):
files['/etc/systemd/system/' + name] = digest(raw(Path('/etc/systemd/system') / name))
# Metadata-only identity of secret trees. Private material is never read.
identities = {}
for name in ('devstate/.env', 'devstate/pki', 'devstate/api-pki', 'devstate/jobs', 'devstate/root-policy'):
path = root / name
paths = [path, *sorted(path.rglob('*'))] if path.is_dir() else [path]
for item in paths:
value = item.lstat()
require(not item.is_symlink() and (stat.S_ISREG(value.st_mode) or stat.S_ISDIR(value.st_mode)), 'preserved state object')
identities[str(item.relative_to(root))] = [value.st_dev, value.st_ino, value.st_uid, value.st_gid,
value.st_mode, value.st_size if stat.S_ISREG(value.st_mode) else None,
value.st_mtime_ns if stat.S_ISREG(value.st_mode) else None]
ingress = run(['systemctl', 'show', 'rmm-server-ingress.socket', '-p', 'ActiveState', '-p', 'UnitFileState']).decode()
require('ActiveState=active' in ingress, 'existing public socket ingress stays active')
return {'files': files, 'protected_metadata': identities, 'ingress': ingress,
'root_policy': closed_root_policy()}
def sql(query):
return json.loads(run(compose + ['exec', '-T', 'postgres', 'psql', '-X', '-qAt', '-v',
'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm'], timeout=30, input=query.encode()))
def database():
# One repeatable-read snapshot, only counts and SHA256 values leave PG.
tables = ('tenants', 'assets', 'operation_plans', 'approval_requests', 'authorizations', 'operations',
'jobs', 'outbox', 'execution_permits', 'job_receipts', 'audit_events', 'operation_submissions',
'live_query_requests', 'memberships', 'tenant_limits', 'oidc_principals', 'package_recipes', 'artifacts')
pairs = []
for table in tables:
pairs.append("'" + table + "',(SELECT json_build_object('count',count(*),'sha256',encode(sha256(convert_to("
"coalesce(string_agg(to_jsonb(t)::text,E'\\n' ORDER BY to_jsonb(t)::text),''),'UTF8')),'hex')) FROM " + table + ' t)')
# Stable device identity excludes expected changing passive Sync/contact.
identity = "(SELECT coalesce(json_agg(q ORDER BY tenant_id,agent_id),'[]'::json) FROM (SELECT tenant_id,agent_id,asset_id,enrollment_generation,assignment_version,certificate_sha256,revoked_at FROM agent_installations) q)"
migrations = "(SELECT json_agg(q ORDER BY version) FROM (SELECT version,description,installed_on,success,encode(checksum,'hex') AS checksum,execution_time FROM _sqlx_migrations) q)"
role = "(SELECT json_build_object('superuser',rolsuper,'bypassrls',rolbypassrls) FROM pg_roles WHERE rolname='rmm_app')"
query = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SELECT json_build_object('history',json_build_object(" + ','.join(pairs) + "),'identities'," + identity + ",'migrations'," + migrations + ",'app_role'," + role + ",'unresolved',(SELECT count(*) FROM jobs WHERE state NOT IN ('succeeded','failed','cancelled','expired'))); COMMIT;"
out = sql(query)
require(out['unresolved'] == 0 and out['app_role'] == {'superuser': False, 'bypassrls': False}, 'settled history and no RLS bypass')
require(all(row['success'] for row in out['migrations']), 'successful SQLx ledger')
return out
def witness(running=True):
current = {'runtime_image': image('rust-discovery-server:local'),
'controls': controls(), 'containers': {name: container(name) for name in ('postgres', 'api', 'gateway')},
'index_sha256': digest(raw(root / 'web/dist/index.html')), 'database': database()}
pg = current['containers']['postgres']
mounts = [m for m in pg['mounts'] if m['Destination'] == '/var/lib/postgresql/data']
require(len(mounts) == 1 and mounts[0]['Type'] == 'volume' and pg['running'], 'existing PG data volume')
value = Path(mounts[0]['Source']).lstat()
require(stat.S_ISDIR(value.st_mode) and not Path(mounts[0]['Source']).is_symlink(), 'same PG data directory')
current['pg_volume'] = {'mount': mounts[0], 'device': value.st_dev, 'inode': value.st_ino}
for name in ('api', 'gateway'):
require(current['containers'][name]['running'] == running, 'fixed application running state')
require(current['containers'][name]['image'] == current['runtime_image'], 'both services run runtime tag')
return current
def before_guard(now, expected, stopped=False):
require(now['runtime_image'] == expected['runtime_image'] == payload['old_image'], 'runtime image CAS')
for key in ('controls', 'index_sha256', 'database', 'pg_volume'):
require(now[key] == expected[key], 'preserved ' + key + ' CAS')
for name in ('postgres', 'api', 'gateway'):
current = dict(now['containers'][name])
old = dict(expected['containers'][name])
if stopped and name != 'postgres':
current.pop('running'); old.pop('running')
require(current == old, 'fixed prior service/container CAS')
def migration_guard(record, after=False):
expected = record['migration_checksums']
require([row['version'] for row in record['baseline']['database']['migrations']] == list(range(1, 14)), 'exact prior migrations 1..13')
for row in record['baseline']['database']['migrations']:
require(row['checksum'] == expected[str(row['version'])], 'frozen old migration checksum')
if after:
now = database()
require(now['migrations'][:13] == record['baseline']['database']['migrations']
and [row['version'] for row in now['migrations']] == list(range(1, 15))
and now['migrations'][13]['checksum'] == expected['14'], 'only additive frozen migration14')
for key in ('history', 'identities', 'app_role', 'unresolved'):
require(now[key] == record['baseline']['database'][key], 'original database rows unchanged')
check = sql("BEGIN READ ONLY; SELECT json_build_object('count',(SELECT count(*) FROM windows_msi_recipes),"
"'rls',(SELECT relrowsecurity AND relforcerowsecurity FROM pg_class WHERE oid='windows_msi_recipes'::regclass),"
"'select',has_table_privilege('rmm_app','windows_msi_recipes','SELECT'),"
"'insert',has_table_privilege('rmm_app','windows_msi_recipes','INSERT'),"
"'update',has_table_privilege('rmm_app','windows_msi_recipes','UPDATE'),"
"'delete',has_table_privilege('rmm_app','windows_msi_recipes','DELETE'),"
"'trigger',EXISTS(SELECT 1 FROM pg_trigger WHERE tgrelid='windows_msi_recipes'::regclass AND tgname='immutable_content' AND tgenabled='O')); COMMIT;")
require(check == {'count': 0, 'rls': True, 'select': True, 'insert': True, 'update': False, 'delete': False, 'trigger': True}, 'empty immutable MSI tenant catalog with FORCE RLS')
return now
def check_source(record):
require(all(digest(raw(folder / 'context' / name)) == value for name, value in record['source_files'].items()), 'frozen build context unchanged')
def https(port, path, host='localhost'):
require((port, host) in ((18443, 'localhost'), (18444, 'localhost'),
(443, 'discovery.ease-technologies.de')), 'fixed role TLS endpoint and name')
ca = root / ('devstate/api-pki/ca.pem' if port == 18443 else 'devstate/pki/ca.pem')
class HTTPS(http.client.HTTPSConnection):
def connect(self):
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', port), timeout=5), server_hostname=self.host)
client = HTTPS(host, port, timeout=5, context=ssl.create_default_context(cafile=str(ca)))
try:
client.request('GET', path)
response = client.getresponse()
body = response.read(2 * 1024 * 1024 + 1)
require(len(body) <= 2 * 1024 * 1024, 'bounded HTTPS body')
return {'status': response.status, 'sha256': digest(body), 'body': body}
finally:
client.close()
def health():
for port in (18443, 18444):
deadline = time.monotonic() + 60
while True:
try:
result = https(port, '/healthz')
require(result['status'] == 200 and result['body'] == b'ok', 'healthy exact server role')
break
except Exception:
require(time.monotonic() < deadline, 'TLS role health deadline')
time.sleep(1)
auth = https(18443, '/auth/config')
require(auth['status'] == 200 and json.loads(auth['body']) ==
{'oidc_configured': False, 'oidc_login_url': None, 'dev_login_enabled': True}, 'honest private dev/test authentication')
for path in ('/', '/auth/config', '/auth/session', '/auth/dev/login', '/mcp', '/api/v1/tenants'):
require(https(443, path, 'discovery.ease-technologies.de')['status'] == 404, 'public gateway has no UI/auth/MCP/admin API')
require(https(443, '/api/v1/job-signing-key', 'discovery.ease-technologies.de')['status'] == 401, 'public signing key requires authenticated agent')
def replace_public(path, data):
require(path.parent.is_dir() and not path.is_symlink(), 'existing served directory, no symlink')
next_path = path.with_name(path.name + '.script-console-' + ident)
once(next_path, data, 0o644)
os.replace(next_path, path)
syncdir(path.parent)
def after_guard(built, index):
now = witness()
require(now['runtime_image'] == built['image_id'], 'exact candidate remains runtime')
for key in ('controls', 'pg_volume'):
require(now[key] == record['baseline'][key], 'preserved post-deploy ' + key)
require(now['index_sha256'] == index, 'exact post-deploy index')
require(now['containers']['postgres'] == record['baseline']['containers']['postgres'], 'same PG container')
for name in ('api', 'gateway'):
before = dict(record['baseline']['containers'][name])
after = dict(now['containers'][name])
for key in ('id', 'image'):
before.pop(key); after.pop(key)
require(before == after, 'same application role, mounts, ports and closed gates')
migration_guard(record, after=True)
# inspect and status never mkdir, upload, launch, migrate, retag or publish.
if action == 'inspect':
return witness()
if action == 'build':
record = payload['record']
require(record['id'] == ident and record['old_image'] == payload['old_image'], 'fixed prepared case')
before_guard(witness(), record['baseline'])
migration_guard(record)
require(record['baseline']['controls']['files']['deploy/Dockerfile'] == record['source_files']['deploy/Dockerfile']
and record['baseline']['controls']['files']['.dockerignore'] == record['source_files']['.dockerignore']
and record['baseline']['controls']['files']['deploy/server.compose.yaml'] == record['source_files']['deploy/server.compose.yaml'], 'preserving unchanged deploy controls')
# Create once before the first write; an incomplete case is status-only.
owned(root, True); owned(root / 'devstate', True)
folder.mkdir(mode=0o700); syncdir(folder.parent)
once(folder / 'intent.json', enc(record))
context = folder / 'context'; context.mkdir(mode=0o700)
archive_raw = base64.b64decode(payload['source_archive'], validate=True)
require(len(archive_raw) <= 20 * 1024 * 1024 and digest(archive_raw) == record['source_archive_sha256'], 'source selection archive')
found = set()
with tarfile.open(fileobj=io.BytesIO(archive_raw), mode='r:') as archive:
for member in archive.getmembers():
name = member.name
require(member.isfile() and name in record['source_files'] and name not in found
and not Path(name).is_absolute() and '..' not in Path(name).parts
and member.size <= 2 * 1024 * 1024, 'fixed complete source selection')
data = archive.extractfile(member).read()
require(digest(data) == record['source_files'][name], 'source member hash')
dest = context / name; dest.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
once(dest, data); found.add(name)
require(found == set(record['source_files']), 'complete frozen source selection')
web = base64.b64decode(payload['web_archive'], validate=True)
require(digest(web) == record['web_archive_sha256'] and len(web) < 2 * 1024 * 1024, 'qualified Web49 archive')
once(folder / 'web.tar', web)
worker = payload['worker_source'].encode()
require(digest(worker) == record['worker_source_sha256'], 'fixed worker source')
once(folder / 'worker.py', worker)
once(folder / 'build-started.json', enc({'id': ident, 'intent_sha256': digest(enc(record))}))
run(['systemd-run', '--unit=rmm-script-build-' + ident, '--property=UMask=0077',
'--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'build-worker'], timeout=15)
return {'id': ident, 'build_launched': True, 'deployed': False, 'lost_reply_action': 'status only'}
if action == 'status':
if not folder.exists():
require(not folder.is_symlink(), 'no case symlink')
return {'id': ident, 'case_present': False, 'read_only': True, 'retry_permitted': False}
owned(folder, True)
receipts = {}
for path in sorted(folder.glob('*.json')):
if path.name != 'intent.json':
try:
receipts[path.name] = load(path)
except Exception as error:
receipts[path.name] = {'read_error_class': type(error).__name__}
result = {'id': ident, 'case_present': True, 'receipts': receipts, 'read_only': True, 'retry_permitted': False}
# Readback must survive an interrupted upload, lost launch reply, failed
# migration or stopped roles. Report partial evidence; never heal it.
checks = {'runtime_image': lambda: image('rust-discovery-server:local'), 'database': database,
'containers': lambda: {name: container(name) for name in ('postgres', 'api', 'gateway')},
'controls': controls, 'index_sha256': lambda: digest(raw(root / 'web/dist/index.html'))}
if (folder / 'intent.json').exists():
checks['intent_sha256'] = lambda: digest(raw(folder / 'intent.json', 8 * 1024 * 1024))
checks['source_unchanged'] = lambda: all((folder / 'context' / name).is_file()
and digest(raw(folder / 'context' / name)) == value
for name, value in load(folder / 'intent.json')['source_files'].items())
def served_web():
values = {}
for name in load(folder / 'intent.json')['web_files']:
got = https(18443, '/' + name if name != 'index.html' else '/')
values[name] = {'status': got['status'], 'sha256': got['sha256']}
return values
checks['private_TLS_web'] = served_web
for name, check in checks.items():
try:
result[name] = check()
except Exception as error:
result[name] = {'read_error_class': type(error).__name__}
for name in ('build', 'deploy'):
path = folder / (name + '.log')
result[name + '_log_bytes'] = path.stat().st_size if path.exists() else None
return result
owned(folder, True)
record = load(folder / 'intent.json')
require(record['id'] == ident and record['old_image'] == payload['old_image'], 'same case intent')
require(digest(raw(folder / 'worker.py')) == record['worker_source_sha256'], 'retained worker unchanged')
def deploy_guard():
check_source(record)
before_guard(witness(), record['baseline'])
migration_guard(record)
built = load(folder / 'build-result.json')
require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'exact successful candidate image')
require(not (folder / 'deploy-started.json').exists(), 'create-once deployment; status only after any launch')
return built
if action == 'deploy':
built = deploy_guard()
once(folder / 'deploy-started.json', enc({'id': ident, 'image_id': built['image_id'], 'intent_sha256': digest(enc(record))}))
run(['systemd-run', '--unit=rmm-script-deploy-' + ident, '--property=UMask=0077',
'--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'deploy-worker'], timeout=15)
return {'id': ident, 'deploy_launched': True, 'completed': False, 'lost_reply_action': 'status only'}
# The retained worker, never caller-supplied shell code, performs each phase
# once. A host-wide lock serializes this runner's build/deployment workers.
require(action in ('build-worker', 'deploy-worker'), 'retained worker action')
lock = os.open(root / 'devstate/.script-console-upgrade-v1.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
phase = 'build' if action == 'build-worker' else 'deploy'
try:
once(folder / (phase + '-worker-started.json'), enc({'id': ident}))
if phase == 'build':
before_guard(witness(), record['baseline'])
check_source(record)
require(not run(['docker', 'image', 'ls', '-q', record['candidate_tag']]).strip(), 'new candidate tag only')
with (folder / 'build.log').open('xb') as log:
result = subprocess.run(['docker', 'build', '--platform=linux/arm64', '--file', str(folder / 'context/deploy/Dockerfile'),
'--tag', record['candidate_tag'], str(folder / 'context')], stdout=log, stderr=subprocess.STDOUT, timeout=3600)
log.flush(); os.fsync(log.fileno())
require(result.returncode == 0, 'native ARM64 image build failed')
check_source(record)
before_guard(witness(), record['baseline'])
outcome = {'passed': True, 'image_id': image(record['candidate_tag']), 'source_files_equal': True,
'runtime_unchanged': True, 'migration_executed': False, 'jobs_or_approvals_created': False}
else:
check_source(record)
built = load(folder / 'build-result.json')
require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'fixed built image')
before_guard(witness(), record['baseline'])
migration_guard(record)
# Intent already durable. Stop both writers before checking/backup;
# failures deliberately leave an incomplete case for operator readback.
run(compose + ['stop', '--timeout', '20', 'api', 'gateway'], timeout=50)
once(folder / '01-services-stopped.json', enc({'id': ident}))
before_guard(witness(running=False), record['baseline'], stopped=True)
active = sql("BEGIN READ ONLY; SELECT json_build_object('other_sessions',count(*)) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND backend_type='client backend' AND application_name<>'pg_isready'; COMMIT;")
require(active['other_sessions'] == 0, 'no other database writer during upgrade')
with (folder / 'postgres-before.dump').open('xb') as dump:
result = subprocess.run(compose + ['exec', '-T', 'postgres', 'pg_dump', '-U', 'postgres', '-d', 'rmm', '--format=custom'],
stdout=dump, stderr=subprocess.PIPE, timeout=120)
dump.flush(); os.fsync(dump.fileno())
require(result.returncode == 0, 'private pre-migration database backup')
once(folder / 'index-before.html', raw(root / 'web/dist/index.html'))
once(folder / '02-backup-complete.json', enc({'id': ident, 'backup_private': True}))
# Parse/check the entire UI archive before publishing any asset.
web_raw = raw(folder / 'web.tar')
require(digest(web_raw) == record['web_archive_sha256'], 'unchanged UI archive')
web = {}
with tarfile.open(fileobj=io.BytesIO(web_raw), mode='r:') as archive:
for member in archive.getmembers():
require(member.isfile() and member.name in record['web_files'] and member.name not in web
and member.size <= 2 * 1024 * 1024, 'exact qualified UI members')
data = archive.extractfile(member).read()
require(digest(data) == record['web_files'][member.name], 'qualified UI bytes')
web[member.name] = data
require(set(web) == set(record['web_files']), 'complete qualified UI')
dist = root / 'web/dist'; owned(dist, True); owned(dist / 'assets', True)
for name, data in web.items():
if name == 'index.html':
continue
dest = dist / name
if dest.exists():
require(digest(raw(dest)) == record['web_files'][name], 'immutable asset filename collision')
else:
once(dest, data, 0o644)
once(folder / '03-assets-published.json', enc({'id': ident, 'index_unchanged': digest(raw(dist / 'index.html')) == record['baseline']['index_sha256']}))
require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'old index CAS before migration')
require(image('rust-discovery-server:local') == record['old_image'], 'runtime tag CAS before transition')
run(['docker', 'tag', built['image_id'], 'rust-discovery-server:local'])
once(folder / '04-runtime-tagged.json', enc({'id': ident, 'image_id': built['image_id']}))
# Named migration container remains inspectable even on failure/lost
# reply. Never remove it or issue a second migration automatically.
with (folder / 'deploy.log').open('xb') as log:
result = subprocess.run(compose + ['run', '--name', 'rmm-script-migrate-' + ident,
'--no-deps', 'migrate'], stdout=log, stderr=subprocess.STDOUT, timeout=120)
log.flush(); os.fsync(log.fileno())
require(result.returncode == 0, 'migration incomplete; inspect retained named container')
migration_guard(record, after=True)
once(folder / '05-migration14-verified.json', enc({'id': ident, 'original_ledger_and_history_equal': True}))
run(compose + ['up', '-d', '--no-deps', '--no-build', 'api', 'gateway'], timeout=60)
health()
after_guard(built, record['baseline']['index_sha256'])
once(folder / '06-backend-ready.json', enc({'id': ident, 'TLS_healthy': True, 'public_admin_routes_closed': True}))
# Directory inode and all old hashed assets survive. Only now switch
# the entry document, atomically and against the exact old index.
require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'final old index CAS')
replace_public(dist / 'index.html', web['index.html'])
for name, expected in record['web_files'].items():
served = https(18443, '/' + name if name != 'index.html' else '/')
require(served['status'] == 200 and served['sha256'] == expected, 'actual private TLS UI byte hash')
after_guard(built, record['web_files']['index.html'])
once(folder / '07-index-published.json', enc({'id': ident, 'web_files': record['web_files']}))
outcome = {'passed': True, 'image_id': built['image_id'], 'migration14': True, 'web_files': record['web_files'],
'prior_history_equal': True, 'jobs_or_approvals_created': False, 'root_system_fault_gates_closed': True,
'production_oidc_qualified': False, 'native_client_acceptance': False}
once(folder / (phase + '-result.json'), enc(outcome))
return outcome
except Exception as error:
# No raw exception/command/DB output can escape to the caller or log.
failure = {'passed': False, 'error_class': type(error).__name__, 'retry_permitted': False,
'frames': [{'function': frame.name, 'line': frame.lineno} for frame in traceback.extract_tb(error.__traceback__)]}
once(folder / (phase + '-failure.json'), enc(failure))
raise RuntimeError('case incomplete; status/readback only') from None
finally:
os.close(lock)
def archive(files):
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode='w:') as tar:
for name, raw in sorted(files.items()):
item = tarfile.TarInfo(name)
item.size = len(raw)
item.mode = 0o644
tar.addfile(item, io.BytesIO(raw))
return stream.getvalue()
def worker_source(ident):
prefix = 'import os,json,subprocess,base64,hashlib,uuid,sys\nfrom pathlib import Path\n'
return (prefix + inspect.getsource(bootstrap.require) + '\n' + inspect.getsource(remote) +
'\nif __name__ == "__main__":\n remote(dict(action=sys.argv[1],id=' + repr(ident) + ',old_image=' + repr(OLD_IMAGE) + '))\n').encode()
def check_support():
require(all(sha((ROOT / name).read_bytes()) == expected for name, expected in SUPPORT.items()), 'pinned SSH support sources')
def prepare(baseline_path):
require(sha(FROZEN.read_bytes()) == FROZEN_SHA, 'frozen158 qualification manifest')
frozen = json.loads(FROZEN.read_bytes())
require(frozen['source_before_after_equal'] and len(frozen['source_files']) == 158
and all(check['exit_code'] == 0 for check in frozen['checks'].values()), 'passing exact qualification')
source_raw = (FROZEN.parent / 'reviewed-sources.tar.gz').read_bytes()
require(sha(source_raw) == frozen['source_archive_sha256'], 'frozen source archive')
sources = {}
with tarfile.open(fileobj=io.BytesIO(source_raw), mode='r:gz') as tar:
for item in tar.getmembers():
require(item.isfile() and item.name in frozen['source_files'] and item.name not in sources
and item.size <= 2 * 1024 * 1024 and not Path(item.name).is_absolute()
and '..' not in Path(item.name).parts, 'frozen source selection')
data = tar.extractfile(item).read()
require(sha(data) == frozen['source_files'][item.name], 'frozen member hash')
sources[item.name] = data
require(set(sources) == set(frozen['source_files']), 'all158 frozen source inputs retained')
for name, expected in EXTRA.items():
path = ROOT / name
require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'separate deployment source pin')
sources[name] = path.read_bytes()
web = {}
for name, expected in WEB.items():
path = ROOT / 'web/dist' / name
require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'qualified Web49 byte pin')
web[name] = path.read_bytes()
# A complete context is manifest-selected, not copied from a live checkout.
for forbidden in ('credentials/', 'devstate/', '.git/', 'target/', 'web/node_modules/'):
require(not any(name.startswith(forbidden) or '/' + forbidden in name for name in sources), 'private/cache path excluded')
require(all(name in sources for name in ('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml',
'crates/oidc-auth/sql/001_oidc_login_states.sql', 'tools/apt-transaction/apt_transaction.py',
'tools/apt-transaction/hook', 'deploy/linux/rmm-apt-helper.service', 'migrations/0014_windows_msi_catalog.sql')), 'Docker COPY/include closure')
baseline_raw = baseline_path.read_bytes()
baseline = json.loads(baseline_raw)
require(baseline['runtime_image'] == OLD_IMAGE and baseline['database']['unresolved'] == 0
and [row['version'] for row in baseline['database']['migrations']] == list(range(1, 14)), 'fresh intended prior runtime/schema')
ident = str(uuid.uuid4())
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
folder = STATE / ident
folder.mkdir(mode=0o700)
data = archive(sources); web_data = archive(web); worker = worker_source(ident)
checksums = {str(int(Path(name).name.split('_', 1)[0])): hashlib.sha384(value).hexdigest()
for name, value in sources.items() if re.fullmatch(r'migrations/\d{4}_[a-z0-9_]+\.sql', name)}
require(set(checksums) == {str(n) for n in range(1, 15)}, 'exact migration1..14 closure')
record = {'schema_version': 1, 'type': 'rmm.script-console.cloud-upgrade.v1', 'id': ident,
'frozen_manifest_sha256': FROZEN_SHA, 'old_image': OLD_IMAGE, 'baseline': baseline,
'baseline_sha256': sha(baseline_raw), 'source_files': {name: sha(value) for name, value in sources.items()},
'source_archive_sha256': sha(data), 'web_files': WEB, 'web_archive_sha256': sha(web_data),
'worker_source_sha256': sha(worker), 'runner_sha256': sha(Path(__file__).read_bytes()),
'support_sha256': SUPPORT, 'migration_checksums': checksums,
'candidate_tag': 'rust-discovery-server:script-v1-' + ident}
for row in baseline['database']['migrations']:
require(row['success'] and row['checksum'] == checksums[str(row['version'])], 'unchanged SQLx1..13 bytes')
for name, raw in (('source.tar', data), ('web.tar', web_data), ('worker.py', worker),
('baseline.json', baseline_raw), ('manifest.json', encoded(record))):
create(folder / name, raw)
return {'id': ident, 'manifest': str(folder / 'manifest.json'), 'manifest_sha256': sha(encoded(record)),
'source_count': len(sources), 'remote_operations': False, 'build_started': False}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('inspect', 'prepare', 'build', 'deploy', 'status'))
parser.add_argument('--id')
parser.add_argument('--baseline', type=Path)
parser.add_argument('--access-root', type=Path, default=ROOT,
help='existing main checkout containing protected server.conf; SSH only')
parser.add_argument('--execute-cloud-build', action='store_true')
parser.add_argument('--execute-cloud-deploy', action='store_true')
args = parser.parse_args()
os.umask(0o077)
check_support()
if args.action == 'prepare':
require(args.baseline is not None, 'explicit saved remote baseline required')
print(json.dumps(prepare(args.baseline)))
return
# Keep the reviewed worktree's source inputs, but reuse the existing main
# checkout's protected SSH transport configuration when explicitly selected.
bootstrap.ROOT = args.access_root.resolve()
bootstrap.access.ROOT = bootstrap.ROOT
config = bootstrap.access.load_config(bootstrap.ROOT / 'credentials/server.conf')
require(config['address'] in ('88.99.80.202', 'discovery.ease-technologies.de')
and config['username'] == 'root' and config.get('port', '22') == '22', 'fixed existing cloud server bootstrap target')
if args.action == 'inspect':
result = bootstrap.rpc('server.conf', remote, {'action': 'inspect', 'id': str(uuid.uuid4()), 'old_image': OLD_IMAGE})
require(result['runtime_image'] == OLD_IMAGE, 'known actual prior server image')
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
path = STATE / ('baseline-' + str(uuid.uuid4()) + '.json')
create(path, encoded(result))
print(json.dumps({'baseline': str(path), 'sha256': sha(encoded(result)), 'read_only_remote': True}))
return
ident = str(uuid.UUID(args.id))
require(ident == args.id, 'canonical case UUID')
folder = STATE / ident
manifest = json.loads((folder / 'manifest.json').read_bytes())
require(manifest['id'] == ident and manifest['runner_sha256'] == sha(Path(__file__).read_bytes())
and manifest['support_sha256'] == SUPPORT and manifest['worker_source_sha256'] == sha(worker_source(ident)), 'retained exact runner/support/worker')
payload = {'action': args.action, 'id': ident, 'old_image': OLD_IMAGE}
if args.action in ('build', 'deploy'):
require(args.execute_cloud_build if args.action == 'build' else args.execute_cloud_deploy, 'explicit bounded cloud action required')
# Local once intent BEFORE SSH: uncertain/lost replies cannot launch twice.
create(folder / (args.action + '-requested.json'), encoded({'id': ident, 'manifest_sha256': sha(encoded(manifest))}))
if args.action == 'build':
payload.update(record=manifest, source_archive=base64.b64encode((folder / 'source.tar').read_bytes()).decode(),
web_archive=base64.b64encode((folder / 'web.tar').read_bytes()).decode(), worker_source=(folder / 'worker.py').read_text())
result = bootstrap.rpc('server.conf', remote, payload)
create(folder / (args.action + '-reply-' + str(uuid.uuid4()) + '.json'), encoded(result))
print(json.dumps(result))
if __name__ == '__main__':
try:
main()
except Exception as error:
print('SCRIPT_CONSOLE_UPGRADE_INCOMPLETE:' + type(error).__name__ + '; preserve case, status only after any request')
raise SystemExit(2) from None
+711
View File
@@ -0,0 +1,711 @@
#!/usr/bin/env python3
"""Version2: exact preserved cloud upgrade, with deterministic Docker mounts.
prepare is offline. inspect/status are remote read-only. build/deploy each launch
one durable, create-once case worker. Lost replies permit status only, not retry.
The notebook never runs an RMM server/client. Existing deployment tools remain
unchanged. See docs/script-console-cloud-upgrade-v2.md before actual use.
"""
import argparse
import base64
import hashlib
import inspect
import io
import json
import os
from pathlib import Path
import re
import tarfile
import uuid
import remote_client_bootstrap as bootstrap
ROOT = Path(__file__).resolve().parents[1]
STATE = ROOT / 'devstate/script-console-cloud-upgrade-v2'
FROZEN = ROOT / 'devstate/script-console-qualification/run-29461892-343f-42f4-b469-88703535425e/manifest.json'
FROZEN_SHA = 'ddd1a78d1cd289b02b1f98572f9652171ec00be84e01ce099fd722d689bcaacd'
OLD_IMAGE = 'sha256:25697f948325a11f755893e16ec5be2da138c3333c8e60f75a18f973d6ac26e2'
EXTRA = {
'deploy/Dockerfile': 'd41b4ec902c11a9e25c22b1c1d3d60d15afc207f2598ecf559cf3bdae4f8e782',
'.dockerignore': 'fdb5cba5967249f5dac35d45c5edf0be2a4e1594c94d04a2ed416d89821ace75',
'deploy/server.compose.yaml': 'd2044b0ee6d584f360990635bec149511d5a0623ec562976541900d751998818',
}
WEB = {
'index.html': '95d6f9058cc713c6af36fbd22ccc60678f081765a853b53d19ac6f0d219804f8',
'assets/index-Be1zm1OZ.css': '916e8870d4e9a9553de452b3ff6c71671bb6a7a22355de0aff432db516a9f3e7',
'assets/index-CTgvl2jr.js': 'f611662042eb4cc44f25c7d5ddc0d2d0562e32be267d3d5fb1db31b0d310f879',
}
SUPPORT = {
'deploy/remote_client_bootstrap.py': '785c43a51dfe3f93e39798dbd7611c353752ae2fc64b22e41ba168e221620d30',
'deploy/vm_access_check.py': '5c8107ea9151a9f498b2530cdc61c1a7a6d745aff853a9b05710f3395838c0dd',
}
PREVIOUS_FAILED_CASE = {
'id': '499e13fb-7433-405d-9a98-0c27ca864246',
'manifest_sha256': '305de92d11b90184589210fd2b0f301ecaf5d9a75467bff5e81c8518588ad416',
'worker_sha256': '6d056e6385ba67c00414de3dc626a51b4f022947bee6381d92908acce3a2b4a9',
'readback_sha256': 'ed2fcd82cdca7692d9c5d122e4cccc1b5aa5a5dc6fec4ac9035ecef647b8e8ef',
'failure_before_docker_build': True,
}
require = bootstrap.require
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def encoded(value):
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def create(path, raw):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw)
stream.flush()
os.fsync(stream.fileno())
fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(fd)
finally:
os.close(fd)
def remote(payload):
"""Self-contained function transported by the existing strict-hostkey RPC."""
import datetime as dt
import fcntl
import http.client
import io
import platform
import socket
import ssl
import stat
import subprocess
import tarfile
import time
import traceback
os.umask(0o077)
root = Path('/opt/rust-discovery')
action = payload['action']
require(os.geteuid() == 0 and platform.machine() == 'aarch64', 'fixed root ARM64 server')
require(action in ('inspect', 'build', 'build-worker', 'deploy', 'deploy-worker', 'status'), 'fixed upgrade action')
ident = str(uuid.UUID(payload['id']))
require(ident == payload['id'], 'canonical case UUID')
folder = root / 'devstate' / ('script-console-upgrade-v2-' + ident)
compose = ['docker', 'compose', '--env-file', str(root / 'devstate/.env'),
'-f', str(root / 'deploy/server.compose.yaml')]
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def enc(value):
return json.dumps(value, sort_keys=True, separators=(',', ':')).encode()
def run(args, timeout=20, **kwargs):
result = subprocess.run(args, capture_output=True, timeout=timeout, **kwargs)
require(result.returncode == 0, 'fixed server command incomplete; inspect private case')
return result.stdout
def owned(path, directory=False):
value = path.lstat()
require(value.st_uid == 0 and not value.st_mode & 0o022 and not path.is_symlink()
and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'root-owned fixed path')
return value
def raw(path, limit=2 * 1024 * 1024):
value = owned(path)
require(value.st_size <= limit and value.st_nlink == 1, 'bounded private regular file')
return path.read_bytes()
def syncdir(path):
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(fd)
finally:
os.close(fd)
def once(path, data, mode=0o600):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(data)
stream.flush()
os.fchmod(stream.fileno(), mode)
os.fsync(stream.fileno())
syncdir(path.parent)
def load(path):
return json.loads(raw(path, 8 * 1024 * 1024))
def image(tag):
# Separate fields avoid ever retrieving environment variables/secrets.
value = run(['docker', 'image', 'inspect', tag, '--format', '{{.Id}} {{.Architecture}} {{.Os}}']).decode().split()
require(len(value) == 3 and value[1:] == ['arm64', 'linux'] and value[0].startswith('sha256:'), 'native ARM64 image')
return value[0]
def closed_root_policy():
# Existing MVP state: flag=true, but this exact historical policy is
# expired. Neither the flag nor the policy is changed by deployment.
directory = root / 'devstate/root-policy'
path = directory / 'policy.json'
parent = directory.lstat(); value = path.lstat()
require(not directory.is_symlink() and stat.S_ISDIR(parent.st_mode)
and parent.st_uid == 10001 and not parent.st_mode & 0o022
and not path.is_symlink() and stat.S_ISREG(value.st_mode)
and value.st_uid == 10001 and stat.S_IMODE(value.st_mode) == 0o600
and value.st_nlink == 1 and 0 < value.st_size <= 65536, 'fixed protected closed Root policy path')
data = path.read_bytes()
expected = 'cf15cbb68b590475df6def28e7dcffb7d88e0ca2d2555a06174fb654eeff05f4'
require(digest(data) == expected, 'exact historical closed Root policy hash')
policy = json.loads(data)
require(isinstance(policy, dict) and set(policy) == {'schema_version', 'profile', 'expires_at', 'scopes'}
and type(policy['schema_version']) is int and policy['schema_version'] == 1
and policy['profile'] == 'development' and isinstance(policy['expires_at'], str)
and isinstance(policy['scopes'], list) and 1 <= len(policy['scopes']) <= 64
and all(isinstance(scope, dict) for scope in policy['scopes']), 'exact legacy Root policy schema')
expiry = dt.datetime.fromisoformat(policy['expires_at'])
require(expiry.utcoffset() is not None and expiry <= dt.datetime.now(dt.timezone.utc), 'expired Root policy denies every scope')
return {'sha256': expected, 'schema_version': 1, 'profile': 'development',
'expires_at': policy['expires_at'], 'closed': True}
def canonical_mounts(mounts):
# Docker does not promise inspection order. Preserve every object and
# attribute, reject ambiguous destinations, then compare canonical lists.
require(isinstance(mounts, list) and all(isinstance(mount, dict)
and isinstance(mount.get('Destination'), str) and mount['Destination'].startswith('/')
and isinstance(mount.get('Type'), str) and bool(mount['Type'])
and isinstance(mount.get('Source'), str) for mount in mounts), 'typed complete Docker mounts')
require(len({mount['Destination'] for mount in mounts}) == len(mounts), 'unique Docker mount destinations')
return sorted(mounts, key=lambda mount: (mount['Destination'], mount['Type'], mount['Source']))
def container(service):
values = run(compose + ['ps', '-a', '-q', service]).decode().split()
require(len(values) == 1, 'one fixed compose service')
cid = values[0]
out = {'id': cid}
for name, fmt in (('image', '.Image'), ('user', '.Config.User'), ('running', '.State.Running'),
('mounts', '.Mounts'), ('ports', '.HostConfig.PortBindings')):
out[name] = json.loads(run(['docker', 'inspect', cid, '--format', '{{json ' + fmt + '}}']))
out['mounts'] = canonical_mounts(out['mounts'])
if service in ('api', 'gateway'):
# Filter inside Docker's template: credential env values never
# enter Python/stdout. No .env, token or private key is opened.
names = ('RMM_PROFILE', 'RMM_DEV_AUTH', 'RMM_SERVER_ROLE', 'RMM_UI_ORIGIN',
'RMM_DEV_ROOT_ENABLED', 'RMM_DEV_SYSTEM_ENABLED', 'RMM_DEV_DELIVERY_FAULTS',
'RMM_DEV_DELIVERY_FAULTS_FILE', 'RMM_DEV_SYSTEM_POLICY_FILE', 'RMM_DEV_ROOT_POLICY_FILE')
select = ' '.join('(eq $key "' + name + '")' for name in names)
fmt = '{{range .Config.Env}}{{$key := index (split . "=") 0}}{{if or ' + select + '}}{{println .}}{{end}}{{end}}'
env = dict(line.split('=', 1) for line in run(['docker', 'inspect', cid, '--format', fmt]).decode().splitlines() if '=' in line)
out['settings'] = {key: env.get(key) for key in names}
require(env.get('RMM_PROFILE') == 'development' and env.get('RMM_DEV_AUTH') == 'true'
and env.get('RMM_SERVER_ROLE') == service and env.get('RMM_UI_ORIGIN') == 'https://localhost:18443'
and env.get('RMM_DEV_ROOT_ENABLED') == 'true'
and env.get('RMM_DEV_ROOT_POLICY_FILE') == '/root-policy/policy.json'
and env.get('RMM_DEV_SYSTEM_ENABLED') is None and env.get('RMM_DEV_DELIVERY_FAULTS') == 'false'
and env.get('RMM_DEV_DELIVERY_FAULTS_FILE') is None and env.get('RMM_DEV_SYSTEM_POLICY_FILE') is None,
'unchanged private dev/test role with Root/SYSTEM/fault gates closed')
mounts = [mount for mount in out['mounts'] if mount['Destination'] == '/root-policy']
require(len(mounts) == 1 and mounts[0]['Type'] == 'bind' and mounts[0]['RW'] is False
and mounts[0]['Source'] == str(root / 'devstate/root-policy'), 'exact read-only Root policy mount')
out['closed_root_policy_sha256'] = closed_root_policy()['sha256']
require(out['user'] == '10001:10001', 'unprivileged application role')
port = '18443' if service == 'api' else '18444'
require(out['ports'] == {'18443/tcp': [{'HostIp': '127.0.0.1', 'HostPort': port}]}, 'loopback-only role')
else:
require(not out['ports'], 'PostgreSQL has no published port')
return out
def controls():
names = ('deploy/Dockerfile', '.dockerignore', 'deploy/server.compose.yaml')
files = {name: digest(raw(root / name)) for name in names}
for name in ('rmm-server-ingress.socket', 'rmm-server-ingress.service'):
files['/etc/systemd/system/' + name] = digest(raw(Path('/etc/systemd/system') / name))
# Metadata-only identity of secret trees. Private material is never read.
identities = {}
for name in ('devstate/.env', 'devstate/pki', 'devstate/api-pki', 'devstate/jobs', 'devstate/root-policy'):
path = root / name
paths = [path, *sorted(path.rglob('*'))] if path.is_dir() else [path]
for item in paths:
value = item.lstat()
require(not item.is_symlink() and (stat.S_ISREG(value.st_mode) or stat.S_ISDIR(value.st_mode)), 'preserved state object')
identities[str(item.relative_to(root))] = [value.st_dev, value.st_ino, value.st_uid, value.st_gid,
value.st_mode, value.st_size if stat.S_ISREG(value.st_mode) else None,
value.st_mtime_ns if stat.S_ISREG(value.st_mode) else None]
ingress = run(['systemctl', 'show', 'rmm-server-ingress.socket', '-p', 'ActiveState', '-p', 'UnitFileState']).decode()
require('ActiveState=active' in ingress, 'existing public socket ingress stays active')
return {'files': files, 'protected_metadata': identities, 'ingress': ingress,
'root_policy': closed_root_policy()}
def sql(query):
return json.loads(run(compose + ['exec', '-T', 'postgres', 'psql', '-X', '-qAt', '-v',
'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm'], timeout=30, input=query.encode()))
def database():
# One repeatable-read snapshot, only counts and SHA256 values leave PG.
tables = ('tenants', 'assets', 'operation_plans', 'approval_requests', 'authorizations', 'operations',
'jobs', 'outbox', 'execution_permits', 'job_receipts', 'audit_events', 'operation_submissions',
'live_query_requests', 'memberships', 'tenant_limits', 'oidc_principals', 'package_recipes', 'artifacts')
pairs = []
for table in tables:
pairs.append("'" + table + "',(SELECT json_build_object('count',count(*),'sha256',encode(sha256(convert_to("
"coalesce(string_agg(to_jsonb(t)::text,E'\\n' ORDER BY to_jsonb(t)::text),''),'UTF8')),'hex')) FROM " + table + ' t)')
# Stable device identity excludes expected changing passive Sync/contact.
identity = "(SELECT coalesce(json_agg(q ORDER BY tenant_id,agent_id),'[]'::json) FROM (SELECT tenant_id,agent_id,asset_id,enrollment_generation,assignment_version,certificate_sha256,revoked_at FROM agent_installations) q)"
migrations = "(SELECT json_agg(q ORDER BY version) FROM (SELECT version,description,installed_on,success,encode(checksum,'hex') AS checksum,execution_time FROM _sqlx_migrations) q)"
role = "(SELECT json_build_object('superuser',rolsuper,'bypassrls',rolbypassrls) FROM pg_roles WHERE rolname='rmm_app')"
query = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SELECT json_build_object('history',json_build_object(" + ','.join(pairs) + "),'identities'," + identity + ",'migrations'," + migrations + ",'app_role'," + role + ",'unresolved',(SELECT count(*) FROM jobs WHERE state NOT IN ('succeeded','failed','cancelled','expired'))); COMMIT;"
out = sql(query)
require(out['unresolved'] == 0 and out['app_role'] == {'superuser': False, 'bypassrls': False}, 'settled history and no RLS bypass')
require(all(row['success'] for row in out['migrations']), 'successful SQLx ledger')
return out
def witness(running=True):
current = {'runtime_image': image('rust-discovery-server:local'),
'controls': controls(), 'containers': {name: container(name) for name in ('postgres', 'api', 'gateway')},
'index_sha256': digest(raw(root / 'web/dist/index.html')), 'database': database()}
pg = current['containers']['postgres']
mounts = [m for m in pg['mounts'] if m['Destination'] == '/var/lib/postgresql/data']
require(len(mounts) == 1 and mounts[0]['Type'] == 'volume' and pg['running'], 'existing PG data volume')
value = Path(mounts[0]['Source']).lstat()
require(stat.S_ISDIR(value.st_mode) and not Path(mounts[0]['Source']).is_symlink(), 'same PG data directory')
current['pg_volume'] = {'mount': mounts[0], 'device': value.st_dev, 'inode': value.st_ino}
for name in ('api', 'gateway'):
require(current['containers'][name]['running'] == running, 'fixed application running state')
require(current['containers'][name]['image'] == current['runtime_image'], 'both services run runtime tag')
return current
def before_guard(now, expected, stopped=False):
require(now['runtime_image'] == expected['runtime_image'] == payload['old_image'], 'runtime image CAS')
for key in ('controls', 'index_sha256', 'database', 'pg_volume'):
require(now[key] == expected[key], 'preserved ' + key + ' CAS')
for name in ('postgres', 'api', 'gateway'):
current = dict(now['containers'][name])
old = dict(expected['containers'][name])
if stopped and name != 'postgres':
current.pop('running'); old.pop('running')
require(current == old, 'fixed prior service/container CAS')
def migration_guard(record, after=False):
expected = record['migration_checksums']
require([row['version'] for row in record['baseline']['database']['migrations']] == list(range(1, 14)), 'exact prior migrations 1..13')
for row in record['baseline']['database']['migrations']:
require(row['checksum'] == expected[str(row['version'])], 'frozen old migration checksum')
if after:
now = database()
require(now['migrations'][:13] == record['baseline']['database']['migrations']
and [row['version'] for row in now['migrations']] == list(range(1, 15))
and now['migrations'][13]['checksum'] == expected['14'], 'only additive frozen migration14')
for key in ('history', 'identities', 'app_role', 'unresolved'):
require(now[key] == record['baseline']['database'][key], 'original database rows unchanged')
check = sql("BEGIN READ ONLY; SELECT json_build_object('count',(SELECT count(*) FROM windows_msi_recipes),"
"'rls',(SELECT relrowsecurity AND relforcerowsecurity FROM pg_class WHERE oid='windows_msi_recipes'::regclass),"
"'select',has_table_privilege('rmm_app','windows_msi_recipes','SELECT'),"
"'insert',has_table_privilege('rmm_app','windows_msi_recipes','INSERT'),"
"'update',has_table_privilege('rmm_app','windows_msi_recipes','UPDATE'),"
"'delete',has_table_privilege('rmm_app','windows_msi_recipes','DELETE'),"
"'trigger',EXISTS(SELECT 1 FROM pg_trigger WHERE tgrelid='windows_msi_recipes'::regclass AND tgname='immutable_content' AND tgenabled='O')); COMMIT;")
require(check == {'count': 0, 'rls': True, 'select': True, 'insert': True, 'update': False, 'delete': False, 'trigger': True}, 'empty immutable MSI tenant catalog with FORCE RLS')
return now
def check_source(record):
require(all(digest(raw(folder / 'context' / name)) == value for name, value in record['source_files'].items()), 'frozen build context unchanged')
def https(port, path, host='localhost'):
require((port, host) in ((18443, 'localhost'), (18444, 'localhost'),
(443, 'discovery.ease-technologies.de')), 'fixed role TLS endpoint and name')
ca = root / ('devstate/api-pki/ca.pem' if port == 18443 else 'devstate/pki/ca.pem')
class HTTPS(http.client.HTTPSConnection):
def connect(self):
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', port), timeout=5), server_hostname=self.host)
client = HTTPS(host, port, timeout=5, context=ssl.create_default_context(cafile=str(ca)))
try:
client.request('GET', path)
response = client.getresponse()
body = response.read(2 * 1024 * 1024 + 1)
require(len(body) <= 2 * 1024 * 1024, 'bounded HTTPS body')
return {'status': response.status, 'sha256': digest(body), 'body': body}
finally:
client.close()
def health():
for port in (18443, 18444):
deadline = time.monotonic() + 60
while True:
try:
result = https(port, '/healthz')
require(result['status'] == 200 and result['body'] == b'ok', 'healthy exact server role')
break
except Exception:
require(time.monotonic() < deadline, 'TLS role health deadline')
time.sleep(1)
auth = https(18443, '/auth/config')
require(auth['status'] == 200 and json.loads(auth['body']) ==
{'oidc_configured': False, 'oidc_login_url': None, 'dev_login_enabled': True}, 'honest private dev/test authentication')
for path in ('/', '/auth/config', '/auth/session', '/auth/dev/login', '/mcp', '/api/v1/tenants'):
require(https(443, path, 'discovery.ease-technologies.de')['status'] == 404, 'public gateway has no UI/auth/MCP/admin API')
require(https(443, '/api/v1/job-signing-key', 'discovery.ease-technologies.de')['status'] == 401, 'public signing key requires authenticated agent')
def replace_public(path, data):
require(path.parent.is_dir() and not path.is_symlink(), 'existing served directory, no symlink')
next_path = path.with_name(path.name + '.script-console-' + ident)
once(next_path, data, 0o644)
os.replace(next_path, path)
syncdir(path.parent)
def after_guard(built, index):
now = witness()
require(now['runtime_image'] == built['image_id'], 'exact candidate remains runtime')
for key in ('controls', 'pg_volume'):
require(now[key] == record['baseline'][key], 'preserved post-deploy ' + key)
require(now['index_sha256'] == index, 'exact post-deploy index')
require(now['containers']['postgres'] == record['baseline']['containers']['postgres'], 'same PG container')
for name in ('api', 'gateway'):
before = dict(record['baseline']['containers'][name])
after = dict(now['containers'][name])
for key in ('id', 'image'):
before.pop(key); after.pop(key)
require(before == after, 'same application role, mounts, ports and closed gates')
migration_guard(record, after=True)
# inspect and status never mkdir, upload, launch, migrate, retag or publish.
if action == 'inspect':
return witness()
if action == 'build':
record = payload['record']
require(record['id'] == ident and record['old_image'] == payload['old_image'], 'fixed prepared case')
before_guard(witness(), record['baseline'])
migration_guard(record)
require(record['baseline']['controls']['files']['deploy/Dockerfile'] == record['source_files']['deploy/Dockerfile']
and record['baseline']['controls']['files']['.dockerignore'] == record['source_files']['.dockerignore']
and record['baseline']['controls']['files']['deploy/server.compose.yaml'] == record['source_files']['deploy/server.compose.yaml'], 'preserving unchanged deploy controls')
# Create once before the first write; an incomplete case is status-only.
owned(root, True); owned(root / 'devstate', True)
folder.mkdir(mode=0o700); syncdir(folder.parent)
once(folder / 'intent.json', enc(record))
context = folder / 'context'; context.mkdir(mode=0o700)
archive_raw = base64.b64decode(payload['source_archive'], validate=True)
require(len(archive_raw) <= 20 * 1024 * 1024 and digest(archive_raw) == record['source_archive_sha256'], 'source selection archive')
found = set()
with tarfile.open(fileobj=io.BytesIO(archive_raw), mode='r:') as archive:
for member in archive.getmembers():
name = member.name
require(member.isfile() and name in record['source_files'] and name not in found
and not Path(name).is_absolute() and '..' not in Path(name).parts
and member.size <= 2 * 1024 * 1024, 'fixed complete source selection')
data = archive.extractfile(member).read()
require(digest(data) == record['source_files'][name], 'source member hash')
dest = context / name; dest.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
once(dest, data); found.add(name)
require(found == set(record['source_files']), 'complete frozen source selection')
web = base64.b64decode(payload['web_archive'], validate=True)
require(digest(web) == record['web_archive_sha256'] and len(web) < 2 * 1024 * 1024, 'qualified Web49 archive')
once(folder / 'web.tar', web)
worker = payload['worker_source'].encode()
require(digest(worker) == record['worker_source_sha256'], 'fixed worker source')
once(folder / 'worker.py', worker)
once(folder / 'build-started.json', enc({'id': ident, 'intent_sha256': digest(enc(record))}))
run(['systemd-run', '--unit=rmm-script-build-' + ident, '--property=UMask=0077',
'--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'build-worker'], timeout=15)
return {'id': ident, 'build_launched': True, 'deployed': False, 'lost_reply_action': 'status only'}
if action == 'status':
if not folder.exists():
require(not folder.is_symlink(), 'no case symlink')
return {'id': ident, 'case_present': False, 'read_only': True, 'retry_permitted': False}
owned(folder, True)
receipts = {}
for path in sorted(folder.glob('*.json')):
if path.name != 'intent.json':
try:
receipts[path.name] = load(path)
except Exception as error:
receipts[path.name] = {'read_error_class': type(error).__name__}
result = {'id': ident, 'case_present': True, 'receipts': receipts, 'read_only': True, 'retry_permitted': False}
# Readback must survive an interrupted upload, lost launch reply, failed
# migration or stopped roles. Report partial evidence; never heal it.
checks = {'runtime_image': lambda: image('rust-discovery-server:local'), 'database': database,
'containers': lambda: {name: container(name) for name in ('postgres', 'api', 'gateway')},
'controls': controls, 'index_sha256': lambda: digest(raw(root / 'web/dist/index.html'))}
if (folder / 'intent.json').exists():
checks['intent_sha256'] = lambda: digest(raw(folder / 'intent.json', 8 * 1024 * 1024))
checks['source_unchanged'] = lambda: all((folder / 'context' / name).is_file()
and digest(raw(folder / 'context' / name)) == value
for name, value in load(folder / 'intent.json')['source_files'].items())
def served_web():
values = {}
for name in load(folder / 'intent.json')['web_files']:
got = https(18443, '/' + name if name != 'index.html' else '/')
values[name] = {'status': got['status'], 'sha256': got['sha256']}
return values
checks['private_TLS_web'] = served_web
for name, check in checks.items():
try:
result[name] = check()
except Exception as error:
result[name] = {'read_error_class': type(error).__name__}
for name in ('build', 'deploy'):
path = folder / (name + '.log')
result[name + '_log_bytes'] = path.stat().st_size if path.exists() else None
return result
owned(folder, True)
record = load(folder / 'intent.json')
require(record['id'] == ident and record['old_image'] == payload['old_image'], 'same case intent')
require(digest(raw(folder / 'worker.py')) == record['worker_source_sha256'], 'retained worker unchanged')
def deploy_guard():
check_source(record)
before_guard(witness(), record['baseline'])
migration_guard(record)
built = load(folder / 'build-result.json')
require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'exact successful candidate image')
require(not (folder / 'deploy-started.json').exists(), 'create-once deployment; status only after any launch')
return built
if action == 'deploy':
built = deploy_guard()
once(folder / 'deploy-started.json', enc({'id': ident, 'image_id': built['image_id'], 'intent_sha256': digest(enc(record))}))
run(['systemd-run', '--unit=rmm-script-deploy-' + ident, '--property=UMask=0077',
'--property=WorkingDirectory=' + str(folder), '/usr/bin/python3', str(folder / 'worker.py'), 'deploy-worker'], timeout=15)
return {'id': ident, 'deploy_launched': True, 'completed': False, 'lost_reply_action': 'status only'}
# The retained worker, never caller-supplied shell code, performs each phase
# once. A host-wide lock serializes this runner's build/deployment workers.
require(action in ('build-worker', 'deploy-worker'), 'retained worker action')
lock = os.open(root / 'devstate/.script-console-upgrade-v1.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
phase = 'build' if action == 'build-worker' else 'deploy'
try:
once(folder / (phase + '-worker-started.json'), enc({'id': ident}))
if phase == 'build':
before_guard(witness(), record['baseline'])
check_source(record)
require(not run(['docker', 'image', 'ls', '-q', record['candidate_tag']]).strip(), 'new candidate tag only')
with (folder / 'build.log').open('xb') as log:
result = subprocess.run(['docker', 'build', '--platform=linux/arm64', '--file', str(folder / 'context/deploy/Dockerfile'),
'--tag', record['candidate_tag'], str(folder / 'context')], stdout=log, stderr=subprocess.STDOUT, timeout=3600)
log.flush(); os.fsync(log.fileno())
require(result.returncode == 0, 'native ARM64 image build failed')
check_source(record)
before_guard(witness(), record['baseline'])
outcome = {'passed': True, 'image_id': image(record['candidate_tag']), 'source_files_equal': True,
'runtime_unchanged': True, 'migration_executed': False, 'jobs_or_approvals_created': False}
else:
check_source(record)
built = load(folder / 'build-result.json')
require(built['passed'] and built['image_id'] == image(record['candidate_tag']), 'fixed built image')
before_guard(witness(), record['baseline'])
migration_guard(record)
# Intent already durable. Stop both writers before checking/backup;
# failures deliberately leave an incomplete case for operator readback.
run(compose + ['stop', '--timeout', '20', 'api', 'gateway'], timeout=50)
once(folder / '01-services-stopped.json', enc({'id': ident}))
before_guard(witness(running=False), record['baseline'], stopped=True)
active = sql("BEGIN READ ONLY; SELECT json_build_object('other_sessions',count(*)) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND backend_type='client backend' AND application_name<>'pg_isready'; COMMIT;")
require(active['other_sessions'] == 0, 'no other database writer during upgrade')
with (folder / 'postgres-before.dump').open('xb') as dump:
result = subprocess.run(compose + ['exec', '-T', 'postgres', 'pg_dump', '-U', 'postgres', '-d', 'rmm', '--format=custom'],
stdout=dump, stderr=subprocess.PIPE, timeout=120)
dump.flush(); os.fsync(dump.fileno())
require(result.returncode == 0, 'private pre-migration database backup')
once(folder / 'index-before.html', raw(root / 'web/dist/index.html'))
once(folder / '02-backup-complete.json', enc({'id': ident, 'backup_private': True}))
# Parse/check the entire UI archive before publishing any asset.
web_raw = raw(folder / 'web.tar')
require(digest(web_raw) == record['web_archive_sha256'], 'unchanged UI archive')
web = {}
with tarfile.open(fileobj=io.BytesIO(web_raw), mode='r:') as archive:
for member in archive.getmembers():
require(member.isfile() and member.name in record['web_files'] and member.name not in web
and member.size <= 2 * 1024 * 1024, 'exact qualified UI members')
data = archive.extractfile(member).read()
require(digest(data) == record['web_files'][member.name], 'qualified UI bytes')
web[member.name] = data
require(set(web) == set(record['web_files']), 'complete qualified UI')
dist = root / 'web/dist'; owned(dist, True); owned(dist / 'assets', True)
for name, data in web.items():
if name == 'index.html':
continue
dest = dist / name
if dest.exists():
require(digest(raw(dest)) == record['web_files'][name], 'immutable asset filename collision')
else:
once(dest, data, 0o644)
once(folder / '03-assets-published.json', enc({'id': ident, 'index_unchanged': digest(raw(dist / 'index.html')) == record['baseline']['index_sha256']}))
require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'old index CAS before migration')
require(image('rust-discovery-server:local') == record['old_image'], 'runtime tag CAS before transition')
run(['docker', 'tag', built['image_id'], 'rust-discovery-server:local'])
once(folder / '04-runtime-tagged.json', enc({'id': ident, 'image_id': built['image_id']}))
# Named migration container remains inspectable even on failure/lost
# reply. Never remove it or issue a second migration automatically.
with (folder / 'deploy.log').open('xb') as log:
result = subprocess.run(compose + ['run', '--name', 'rmm-script-migrate-' + ident,
'--no-deps', 'migrate'], stdout=log, stderr=subprocess.STDOUT, timeout=120)
log.flush(); os.fsync(log.fileno())
require(result.returncode == 0, 'migration incomplete; inspect retained named container')
migration_guard(record, after=True)
once(folder / '05-migration14-verified.json', enc({'id': ident, 'original_ledger_and_history_equal': True}))
run(compose + ['up', '-d', '--no-deps', '--no-build', 'api', 'gateway'], timeout=60)
health()
after_guard(built, record['baseline']['index_sha256'])
once(folder / '06-backend-ready.json', enc({'id': ident, 'TLS_healthy': True, 'public_admin_routes_closed': True}))
# Directory inode and all old hashed assets survive. Only now switch
# the entry document, atomically and against the exact old index.
require(digest(raw(dist / 'index.html')) == record['baseline']['index_sha256'], 'final old index CAS')
replace_public(dist / 'index.html', web['index.html'])
for name, expected in record['web_files'].items():
served = https(18443, '/' + name if name != 'index.html' else '/')
require(served['status'] == 200 and served['sha256'] == expected, 'actual private TLS UI byte hash')
after_guard(built, record['web_files']['index.html'])
once(folder / '07-index-published.json', enc({'id': ident, 'web_files': record['web_files']}))
outcome = {'passed': True, 'image_id': built['image_id'], 'migration14': True, 'web_files': record['web_files'],
'prior_history_equal': True, 'jobs_or_approvals_created': False, 'root_system_fault_gates_closed': True,
'production_oidc_qualified': False, 'native_client_acceptance': False}
once(folder / (phase + '-result.json'), enc(outcome))
return outcome
except Exception as error:
# No raw exception/command/DB output can escape to the caller or log.
failure = {'passed': False, 'error_class': type(error).__name__, 'retry_permitted': False,
'frames': [{'function': frame.name, 'line': frame.lineno} for frame in traceback.extract_tb(error.__traceback__)]}
once(folder / (phase + '-failure.json'), enc(failure))
raise RuntimeError('case incomplete; status/readback only') from None
finally:
os.close(lock)
def archive(files):
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode='w:') as tar:
for name, raw in sorted(files.items()):
item = tarfile.TarInfo(name)
item.size = len(raw)
item.mode = 0o644
tar.addfile(item, io.BytesIO(raw))
return stream.getvalue()
def worker_source(ident):
prefix = 'import os,json,subprocess,base64,hashlib,uuid,sys\nfrom pathlib import Path\n'
return (prefix + inspect.getsource(bootstrap.require) + '\n' + inspect.getsource(remote) +
'\nif __name__ == "__main__":\n remote(dict(action=sys.argv[1],id=' + repr(ident) + ',old_image=' + repr(OLD_IMAGE) + '))\n').encode()
def check_support():
require(all(sha((ROOT / name).read_bytes()) == expected for name, expected in SUPPORT.items()), 'pinned SSH support sources')
def prepare(baseline_path):
require(sha(FROZEN.read_bytes()) == FROZEN_SHA, 'frozen158 qualification manifest')
frozen = json.loads(FROZEN.read_bytes())
require(frozen['source_before_after_equal'] and len(frozen['source_files']) == 158
and all(check['exit_code'] == 0 for check in frozen['checks'].values()), 'passing exact qualification')
source_raw = (FROZEN.parent / 'reviewed-sources.tar.gz').read_bytes()
require(sha(source_raw) == frozen['source_archive_sha256'], 'frozen source archive')
sources = {}
with tarfile.open(fileobj=io.BytesIO(source_raw), mode='r:gz') as tar:
for item in tar.getmembers():
require(item.isfile() and item.name in frozen['source_files'] and item.name not in sources
and item.size <= 2 * 1024 * 1024 and not Path(item.name).is_absolute()
and '..' not in Path(item.name).parts, 'frozen source selection')
data = tar.extractfile(item).read()
require(sha(data) == frozen['source_files'][item.name], 'frozen member hash')
sources[item.name] = data
require(set(sources) == set(frozen['source_files']), 'all158 frozen source inputs retained')
for name, expected in EXTRA.items():
path = ROOT / name
require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'separate deployment source pin')
sources[name] = path.read_bytes()
web = {}
for name, expected in WEB.items():
path = ROOT / 'web/dist' / name
require(not path.is_symlink() and sha(path.read_bytes()) == expected, 'qualified Web49 byte pin')
web[name] = path.read_bytes()
# A complete context is manifest-selected, not copied from a live checkout.
for forbidden in ('credentials/', 'devstate/', '.git/', 'target/', 'web/node_modules/'):
require(not any(name.startswith(forbidden) or '/' + forbidden in name for name in sources), 'private/cache path excluded')
require(all(name in sources for name in ('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml',
'crates/oidc-auth/sql/001_oidc_login_states.sql', 'tools/apt-transaction/apt_transaction.py',
'tools/apt-transaction/hook', 'deploy/linux/rmm-apt-helper.service', 'migrations/0014_windows_msi_catalog.sql')), 'Docker COPY/include closure')
baseline_raw = baseline_path.read_bytes()
baseline = json.loads(baseline_raw)
require(baseline['runtime_image'] == OLD_IMAGE and baseline['database']['unresolved'] == 0
and [row['version'] for row in baseline['database']['migrations']] == list(range(1, 14)), 'fresh intended prior runtime/schema')
ident = str(uuid.uuid4())
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
folder = STATE / ident
folder.mkdir(mode=0o700)
data = archive(sources); web_data = archive(web); worker = worker_source(ident)
checksums = {str(int(Path(name).name.split('_', 1)[0])): hashlib.sha384(value).hexdigest()
for name, value in sources.items() if re.fullmatch(r'migrations/\d{4}_[a-z0-9_]+\.sql', name)}
require(set(checksums) == {str(n) for n in range(1, 15)}, 'exact migration1..14 closure')
record = {'schema_version': 1, 'type': 'rmm.script-console.cloud-upgrade.v2', 'id': ident,
'frozen_manifest_sha256': FROZEN_SHA, 'old_image': OLD_IMAGE, 'baseline': baseline,
'baseline_sha256': sha(baseline_raw), 'source_files': {name: sha(value) for name, value in sources.items()},
'source_archive_sha256': sha(data), 'web_files': WEB, 'web_archive_sha256': sha(web_data),
'worker_source_sha256': sha(worker), 'runner_sha256': sha(Path(__file__).read_bytes()),
'support_sha256': SUPPORT, 'migration_checksums': checksums,
'candidate_tag': 'rust-discovery-server:script-v2-' + ident}
record['supersedes_failed_case'] = PREVIOUS_FAILED_CASE
for row in baseline['database']['migrations']:
require(row['success'] and row['checksum'] == checksums[str(row['version'])], 'unchanged SQLx1..13 bytes')
for name, raw in (('source.tar', data), ('web.tar', web_data), ('worker.py', worker),
('baseline.json', baseline_raw), ('manifest.json', encoded(record))):
create(folder / name, raw)
return {'id': ident, 'manifest': str(folder / 'manifest.json'), 'manifest_sha256': sha(encoded(record)),
'source_count': len(sources), 'remote_operations': False, 'build_started': False}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('inspect', 'prepare', 'build', 'deploy', 'status'))
parser.add_argument('--id')
parser.add_argument('--baseline', type=Path)
parser.add_argument('--access-root', type=Path, default=ROOT,
help='existing main checkout containing protected server.conf; SSH only')
parser.add_argument('--execute-cloud-build', action='store_true')
parser.add_argument('--execute-cloud-deploy', action='store_true')
args = parser.parse_args()
os.umask(0o077)
check_support()
if args.action == 'prepare':
require(args.baseline is not None, 'explicit saved remote baseline required')
print(json.dumps(prepare(args.baseline)))
return
# Keep the reviewed worktree's source inputs, but reuse the existing main
# checkout's protected SSH transport configuration when explicitly selected.
bootstrap.ROOT = args.access_root.resolve()
bootstrap.access.ROOT = bootstrap.ROOT
config = bootstrap.access.load_config(bootstrap.ROOT / 'credentials/server.conf')
require(config['address'] in ('88.99.80.202', 'discovery.ease-technologies.de')
and config['username'] == 'root' and config.get('port', '22') == '22', 'fixed existing cloud server bootstrap target')
if args.action == 'inspect':
result = bootstrap.rpc('server.conf', remote, {'action': 'inspect', 'id': str(uuid.uuid4()), 'old_image': OLD_IMAGE})
require(result['runtime_image'] == OLD_IMAGE, 'known actual prior server image')
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
path = STATE / ('baseline-' + str(uuid.uuid4()) + '.json')
create(path, encoded(result))
print(json.dumps({'baseline': str(path), 'sha256': sha(encoded(result)), 'read_only_remote': True}))
return
ident = str(uuid.UUID(args.id))
require(ident == args.id, 'canonical case UUID')
folder = STATE / ident
manifest = json.loads((folder / 'manifest.json').read_bytes())
require(manifest['id'] == ident and manifest['runner_sha256'] == sha(Path(__file__).read_bytes())
and manifest['support_sha256'] == SUPPORT and manifest['worker_source_sha256'] == sha(worker_source(ident)), 'retained exact runner/support/worker')
payload = {'action': args.action, 'id': ident, 'old_image': OLD_IMAGE}
if args.action in ('build', 'deploy'):
require(args.execute_cloud_build if args.action == 'build' else args.execute_cloud_deploy, 'explicit bounded cloud action required')
# Local once intent BEFORE SSH: uncertain/lost replies cannot launch twice.
create(folder / (args.action + '-requested.json'), encoded({'id': ident, 'manifest_sha256': sha(encoded(manifest))}))
if args.action == 'build':
payload.update(record=manifest, source_archive=base64.b64encode((folder / 'source.tar').read_bytes()).decode(),
web_archive=base64.b64encode((folder / 'web.tar').read_bytes()).decode(), worker_source=(folder / 'worker.py').read_text())
result = bootstrap.rpc('server.conf', remote, payload)
create(folder / (args.action + '-reply-' + str(uuid.uuid4()) + '.json'), encoded(result))
print(json.dumps(result))
if __name__ == '__main__':
try:
main()
except Exception as error:
print('SCRIPT_CONSOLE_UPGRADE_INCOMPLETE:' + type(error).__name__ + '; preserve case, status only after any request')
raise SystemExit(2) from None
+25
View File
@@ -196,3 +196,28 @@ Keine Agent-/Helperinstallation, Änderung bestehender Dienste/Policies/PKI/
Datenbanken, veröffentlichte Capability oder WebUI-Genehmigung durch diesen
Unitlauf. Der oben ursprünglich geplante Unitaufruf ist damit abgeschlossen;
Bootstrap, tatsächlicher FullMaintenancejob und Windowsabnahme bleiben offen.
## Verbindlicher Generationsentscheid vor dem Bootstrap
Ein eigener neuer Fullhelper bei weiterlaufendem altem Generalhelper reicht
nicht: Alle aktiven Leser des gemeinsamen Root-Ledgers müssen neue
`privileged_script_profile`-Jobbytes unabhängig dekodieren und prüfen können.
Alte Decoder verweigern selbst geschlossene neue Zeilen und können beim
Snapshot/Sync einen sticky Fault schreiben. Deshalb neuer Agent und aktive
General-/Fullhelper gemeinsam auf die neue qualifizierte Generation; bestehende
Generalunit/Sandbox/Config/Policy-/Pinbytes unverändert. APT bleibt disabled/
inactive, ohne Endpoint und mit geschlossenen Scopes. Seine aktuelle Generation
war bereits laut vorherigem Runtimeindex nicht qualifiziert; historische
Abnahmen bleiben erhalten. Vor späterer Wiedereröffnung eigener APT-Stackpin-
/Generationnachweis, kein alter Helper gegen neue Ledgerzeilen.
Der Rootprozess hat die84 native Cloud-Linux-Units tatsächlich ausgeführt:
Fall `7b61ce46-7913-492b-9c6b-414a0c2b6ffc`, Proof-SHA256
`a831a70a3cf76a3e8a9bf5820dcca7100d2ef9860dd9388539fe9c13f949f470`,
84PASS und5 ausdrücklich ausgelassene DB-Fälle. Diese5 sind im isolierten lokalen
45er-Lauf bestanden. Keine RMM-Jobs oder neue Capabilities aus diesem Unitlauf;
Runtimeaustausch, tatsächlicher signierter Abgleich und genehmigter Fulljob bleiben
eigene noch offene Schritte.
Details und Nach-MVP-Entdeckung:
[script-console-helper-generations.md](../../docs/script-console-helper-generations.md).
+245
View File
@@ -0,0 +1,245 @@
#!/usr/bin/env python3
"""Prepare the exact preserving bootstrap intent and a READ-ONLY target CAS.
There is deliberately no install/stop/start/policy operation in this preflight.
A runtime index is provenance, never a fresh target observation. Root reviews
this source before invoking snapshot; no client is installed on the notebook.
"""
import argparse
import base64
import hashlib
import inspect
import json
from pathlib import Path
import uuid
ROOT = Path(__file__).resolve().parents[1]
ACCESS_ROOT = Path('/home/kevin/Schreibtisch/Coding/rust-discovery')
SUPPORT = {
'deploy/script_console_acceptance.py': '2b3ba8d548d18627643a9f73c9a4f7bacce4ac2cd5495745c6760d088e078145',
'deploy/windows_msi_runtime_acceptance.py': '833ad8806c90a19342aa6fbc71c33e81975caffb1746550e9b0fbdca7fa0c7ab',
'deploy/vm_access_check.py': '5c8107ea9151a9f498b2530cdc61c1a7a6d745aff853a9b05710f3395838c0dd',
}
for name, digest in SUPPORT.items():
path = ROOT / name
if path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != digest:
raise RuntimeError('Pinned preflight support source changed')
import script_console_acceptance as native
support = native.support
INDEX = ACCESS_ROOT / 'devstate/cloud-shared-core-current-runtime.json'
INDEX_SHA = 'bf058d30ec7252071acbf39aa09007a083a0d237054a1f0815caf8e8510ce145'
PRIOR = ACCESS_ROOT / 'devstate/shared-core/runtime-upgrade-v2/a2423612-721f-4a6c-a832-b9eef549c41e/intent.json'
PRIOR_SHA = '4a1792151d95b26245464ed9df22a439d0979026c5d88052926b76985495bfc4'
NATIVE = ROOT / 'devstate/script-console-native/7b61ce46-7913-492b-9c6b-414a0c2b6ffc/passed-e214cd9b-21ac-48ce-82dc-d10364329d04.json'
NATIVE_SHA = 'a831a70a3cf76a3e8a9bf5820dcca7100d2ef9860dd9388539fe9c13f949f470'
STATE = ROOT / 'devstate/script-console-bootstrap'
def require(value, code):
if not value:
raise RuntimeError(code)
def snapshot_vm(p):
import base64
import contextlib
import hashlib
import json
import os
from pathlib import Path
import sqlite3
import stat
import subprocess
require(os.geteuid() == 0 and os.uname().machine == 'x86_64'
and p['type'] == 'rmm.script-console.preserving-preflight-v1', 'fixed root preflight')
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == p['boot_id'], 'same actual boot')
def sha(data):
return hashlib.sha256(data).hexdigest()
def file(path, owner=0):
path = Path(path); s = path.lstat()
require(not stat.S_ISLNK(s.st_mode) and stat.S_ISREG(s.st_mode)
and s.st_uid == owner and s.st_nlink == 1 and not s.st_mode & 0o022, 'safe fixed file')
return dict(device=s.st_dev, inode=s.st_ino, mode=stat.S_IMODE(s.st_mode), uid=s.st_uid,
gid=s.st_gid, bytes=s.st_size, sha256=sha(path.read_bytes()))
def service(name):
result = subprocess.run(['systemctl', 'show', name, '-p', 'ActiveState', '-p', 'MainPID',
'-p', 'UnitFileState', '-p', 'LoadState'], capture_output=True, check=False, timeout=10)
value=dict(line.split('=',1) for line in result.stdout.decode().splitlines())
if name == 'rmm-full-maintenance-helper.service':
require(result.returncode in (0,1) and value.get('LoadState') == 'not-found'
and value.get('ActiveState') == 'inactive' and value.get('MainPID') == '0',
'only exact new absent unit admits rc0/1')
else:
require(result.returncode == 0, 'known unit query strictly succeeds')
return value
units = {name: service(name) for name in ('rmm-agent-remote.service', 'rmm-helper-cloud.service',
'rmm-apt-helper.service', 'rmm-agent.service', 'rmm-helper-dev-test.service',
'rmm-full-maintenance-helper.service')}
require(all(units[n]['ActiveState'] == 'active' for n in ('rmm-agent-remote.service', 'rmm-helper-cloud.service')),
'known original active pair')
require(all(units[n]['ActiveState'] == 'inactive' and units[n]['UnitFileState'] == 'disabled'
for n in ('rmm-apt-helper.service', 'rmm-agent.service', 'rmm-helper-dev-test.service')),
'APT and legacy daemons remain disabled/inactive')
require(units['rmm-full-maintenance-helper.service']['LoadState'] == 'not-found', 'new Full unit absent')
processes = {}
for name, key, uid in (('rmm-agent-remote.service', 'agent', 993), ('rmm-helper-cloud.service', 'helper', 0)):
pid = int(units[name]['MainPID']); require(pid > 1, 'actual live service PID')
proc=Path('/proc',str(pid))
def start_ticks():
raw=(proc/'stat').read_text(); _,fields=raw.rsplit(') ',1)
return int(fields.split()[19])
before_ticks=start_ticks()
status = (proc/'status').read_text().splitlines()
ids = {line.split(':')[0]: list(map(int, line.split()[1:])) for line in status if line.startswith(('Uid:', 'Gid:'))}
require(ids['Uid'] == [uid] * 4 and ids['Gid'] == [984] * 4, 'actual qualified account identity')
image = Path('/proc', str(pid), 'exe')
image_hash = sha(image.read_bytes())
require(image_hash == p['old_artifacts'][key], 'actual running original code generation')
require(start_ticks() == before_ticks and int(service(name)['MainPID']) == pid,
'same service PID and startticks through code observation')
processes[key] = dict(pid=pid, start_ticks=before_ticks, uids=ids['Uid'], gids=ids['Gid'], sha256=image_hash)
fixed = {
'/usr/local/libexec/rmm-agent-remote': p['old_artifacts']['agent'],
'/usr/local/libexec/rmm-privileged-helper': p['old_artifacts']['helper'],
'/usr/lib/rmm-apt-transaction/apt_transaction.py': p['old_artifacts']['provider'],
'/etc/rmm-remote/helper-general.json': p['general_config_sha256'],
'/etc/rmm-remote/general-helper-proof.json': p['general_pin_sha256'],
'/etc/systemd/system/rmm-agent-remote.service': p['agent_unit_sha256'],
'/etc/systemd/system/rmm-helper-cloud.service': p['general_unit_sha256'],
}
controls = {}
for path, digest in fixed.items():
value = file(path); require(value['sha256'] == digest, 'exact original runtime/control CAS')
controls[path] = value
for path in ('/etc/rmm/apt-helper.json', '/etc/rmm-remote/apt-helper-proof.json',
'/etc/systemd/system/rmm-apt-helper.service', '/usr/lib/rmm-apt-transaction/hook',
'/var/lib/dpkg/status', '/usr/share/rmm-mvp-test/version'):
controls[path] = file(path)
environment = subprocess.run(['systemctl', 'show', 'rmm-agent-remote.service', '-p', 'Environment', '--value'],
capture_output=True, check=True, timeout=10).stdout
require(b'RMM_APT_HELPER_SOCKET=' not in environment and b'RMM_APT_HELPER_PROOF_FILE=' not in environment,
'APT endpoint remains absent')
# Preserve the exact previously proven hardlink exception, never generalize it.
aliases = [Path('/var/lib/rmm-apt-transaction') / name for name in
('acceptance-rmm-apt-fixture-update-1.1.json', 'pending-acceptance-rmm-apt-fixture-update-1.1.json')]
alias_identity = None
for path in aliases:
s = path.lstat()
require(not path.is_symlink() and stat.S_ISREG(s.st_mode) and s.st_nlink == 2
and stat.S_IMODE(s.st_mode) == 0o600 and s.st_uid == 0 and s.st_gid == 984
and sha(path.read_bytes()) == p['apt_alias_sha256'], 'exact known historical APT alias')
identity = (s.st_dev, s.st_ino)
require(identity == (2049,1028362), 'unchanged independently proven APT alias inode')
require(alias_identity is None or identity == alias_identity, 'same original alias inode')
alias_identity = identity
databases = {}
for path, expected in p['database_inodes'].items():
path = Path(path); s = path.lstat(); owner = 0 if path.parent.name == 'rmm-helper' else 993
require(not path.is_symlink() and stat.S_ISREG(s.st_mode) and s.st_nlink == 1 and s.st_uid == owner
and [s.st_dev, s.st_ino] == expected, 'same original database inode/owner')
require(all(not Path(str(path) + suffix).exists() and not Path(str(path) + suffix).is_symlink()
for suffix in ('-wal', '-shm', '-journal')), 'settled DELETE journal required')
with contextlib.closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)) as db:
db.execute('PRAGMA query_only=ON'); db.execute('BEGIN')
require(db.execute('PRAGMA journal_mode').fetchone()[0] == 'delete'
and db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'actual query-only journal integrity')
schemas = dict(db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name"))
if path.name == 'journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0,
'no unfinished agent attempt')
require(db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no agent fault')
if path.parent.name == 'rmm-helper':
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
row_hashes = {r[0]: sha(json.dumps([dict(sha256=sha(v), size=len(v)) if isinstance(v, bytes) else v for v in r], separators=(',', ':')).encode()) for r in rows}
require(row_hashes == p['root_rows'] and len(rows) == 18 and all(r[5] is not None for r in rows), 'exact original closed Root18')
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no helper fault')
databases[str(path)] = dict(device=s.st_dev, inode=s.st_ino, schema_sha256=sha(json.dumps(schemas, sort_keys=True).encode()), integrity='ok')
require([path.stat().st_dev, path.stat().st_ino] == expected, 'journal was not replaced during read')
binding = json.loads(Path('/var/lib/rmm-agent-remote/binding.json').read_bytes())
require(all(binding[k] == v for k, v in p['target'].items())
and all(binding[k] == p['helper_identity'][k] for k in ('enrollment_generation','assignment_version'))
and sha(bytes.fromhex(binding['public_key_hex'])) == p['helper_identity']['signing_key_id'], 'same enrolled target/generations/public signing identity')
config = json.loads(Path('/etc/rmm-remote/helper-general.json').read_bytes())
require(config['agent_uid'] == 993 and config['socket_gid'] == 984
and config['state'] == '/var/lib/rmm-helper' and config['allow_reboot'] is False, 'unchanged existing helper identity/profile')
for path in ('/etc/rmm-remote/helper-full-maintenance.json','/etc/rmm-remote/full-maintenance-helper-proof.json',
'/etc/systemd/system/rmm-agent-remote.service.d/60-full-maintenance.conf'):
require(not os.path.lexists(path), 'new Full paths absent; do not overwrite partial bootstrap')
full = dict(profile='full_maintenance_v1', binding=config['binding'], agent_uid=993, socket_gid=984,
socket='/run/rmm-helper-maintenance/control.sock', state='/var/lib/rmm-helper',
allowed_services=[], allow_reboot=False, allow_root_scripts=True,
allowed_script_profiles=['full_maintenance_v1'], helper_binary_sha256=p['new_artifacts']['helper'])
if config.get('signing_key_file') is not None:
full['signing_key_file'] = config['signing_key_file']
full_bytes = json.dumps(full, separators=(',', ':')).encode()
return dict(type=p['type'], id=p['id'], boot_id=p['boot_id'], target=p['target'], processes=processes,
controls=controls, databases=databases, root_consumed=18, root_unresolved=0,
apt_endpoint_enabled=False, apt_current_runtime_qualified=False,
full_config_candidate=dict(path='/etc/rmm-remote/helper-full-maintenance.json',
mode=0o600, uid=0, gid=0, bytes_b64=base64.b64encode(full_bytes).decode(), sha256=sha(full_bytes)),
server_gate_verified=False, fresh_target_cas=True, install_authorized=False,
services_changed=False, product_state_writes=False, rmm_jobs=False, grants_created=False,
capabilities_activated=False, approvals=False, reboot_requested=False, package_effects=False)
def local_intent(case_id=None):
import os
os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
support.private(STATE, True)
for path, digest in ((INDEX, INDEX_SHA), (PRIOR, PRIOR_SHA), (NATIVE, NATIVE_SHA)):
require(support.sha(path) == digest, 'exact prior provenance/native proof')
index = support.load(INDEX); old = support.load(PRIOR); proof = support.load(NATIVE)
require(index['verified_passive_runtime_upgrade'] is True and index['apt_endpoint_enabled'] is False
and index['apt_current_runtime_qualified'] is False and index['root_consumed'] == 18
and index['root_unresolved'] == 0 and index['central_counts'] == dict(jobs=30, outbox=30, permits=26, unresolved=0), 'known closed prior index, not a fresh target proof')
require(proof['verified'] is True and proof['manifest_sha256'] == native.MANIFEST_SHA
and proof['native']['passed'] == 84 and proof['native']['ignored_database_cases'] == 5, 'actual Linux-first84 proof')
manifest = support.load(native.PACK / 'manifest.json')
require(support.sha(native.PACK / 'manifest.json') == native.MANIFEST_SHA
and all(support.sha(ROOT / name) == value for name, value in manifest['source_files'].items()), 'frozen158 source qualification')
if case_id is not None:
require(str(uuid.UUID(case_id)) == case_id, 'canonical original local case')
folder = STATE / case_id; intent = support.load(folder / 'intent.json')
require(intent['runner_sha256'] == support.sha(__file__) and intent['support_files'] == SUPPORT
and intent['index_sha256'] == INDEX_SHA and intent['manifest_sha256'] == native.MANIFEST_SHA,
'original immutable preflight intent')
return folder, intent
case_id = str(uuid.uuid4()); folder = STATE / case_id; folder.mkdir(mode=0o700)
intent = dict(type='rmm.script-console.preserving-preflight-v1', id=case_id, runner_sha256=support.sha(__file__),
support_files=SUPPORT, index_sha256=INDEX_SHA, previous_intent_sha256=PRIOR_SHA,
manifest_sha256=native.MANIFEST_SHA, native_proof_sha256=NATIVE_SHA,
boot_id=index['boot_id'], target=index['target'], helper_identity=old['general_pin']['identity'], old_artifacts=index['artifact_sha256'],
new_artifacts={key: manifest['artifacts'][name]['sha256'] for key, name in
(('agent', 'rmm-agent'), ('helper', 'rmm-privileged-helper'))},
root_rows=old['root_rows'], database_inodes=old['database_inodes'], apt_alias_sha256=old['alias_sha256'],
general_config_sha256=old['config_sha256'], general_pin_sha256=old['pin_sha256'],
agent_unit_sha256=old['agent_unit_sha256'], general_unit_sha256=old['general_unit_sha256'],
require_fresh_target_cas=True, require_fresh_server_gate=True, install_authorized=False)
support.save(folder / 'intent.json', intent)
return folder, intent
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('mode', choices=('prepare-local', 'snapshot', 'read-local'))
parser.add_argument('--case')
args = parser.parse_args()
require((args.mode == 'prepare-local') == (args.case is None), 'prepare new local intent; snapshot/read need original case')
folder, intent = local_intent(args.case)
if args.mode == 'snapshot':
for name, digest in SUPPORT.items():
require(support.sha(ROOT / name) == digest, 'support unchanged before target read')
code = inspect.getsource(require) + '\n' + inspect.getsource(snapshot_vm)
code += '\ntry:\n print(__import__("json").dumps(snapshot_vm(' + repr(intent) + ')))\nexcept Exception as e:\n print(__import__("json").dumps(dict(bootstrap_error=True,error_class=type(e).__name__)))\n'
prefix = support.access_settings(ACCESS_ROOT, 'linux')
report = support.transport(prefix, 'python3 -', code.encode(), folder, timeout=60)
support.save(folder / ('snapshot-' + str(uuid.uuid4()) + '.json'), report)
require(report['fresh_target_cas'] is True and report['install_authorized'] is False
and all(report.get(k) is False for k in ('services_changed', 'product_state_writes', 'rmm_jobs',
'grants_created', 'capabilities_activated', 'approvals', 'reboot_requested', 'package_effects')), 'strict read-only preflight scope')
print(json.dumps(dict(case_id=intent['id'], intent=str(folder / 'intent.json'), sha256=support.sha(folder / 'intent.json'),
mode=args.mode, install_authorized=False, target_called=args.mode == 'snapshot')))
+351
View File
@@ -0,0 +1,351 @@
#!/usr/bin/env python3
"""Reviewable preserving Linux runtime installation; parent executes only.
All active shared-ledger readers become the same new generation. General rights
and controls remain byte-identical. APT stays already closed/unqualified. No RMM
job/approval or automatic install retry. This source never installs the notebook.
"""
import argparse
import base64
import hashlib
import inspect
import json
import os
from pathlib import Path
import uuid
ROOT = Path(__file__).resolve().parents[1]
PREFLIGHT_SHA = '0e210b857d3f25e0ff228bbbd4a6533dadd91c9d21dbcba5923b26c2fe72eec9'
if hashlib.sha256((ROOT / 'deploy/script_console_bootstrap.py').read_bytes()).hexdigest() != PREFLIGHT_SHA:
raise RuntimeError('Reviewed preflight source changed')
import script_console_bootstrap as pre
support = pre.support
STATE = ROOT / 'devstate/script-console-install'
REMOTE = '/var/lib/rmm-bootstrap-history/script-console-v1-'
def require(value, code):
if not value:
raise RuntimeError(code)
def server_gate(p):
import datetime as dt
import hashlib
import json
from pathlib import Path
import subprocess
import uuid
tenant = str(uuid.UUID(p['target']['tenant_id'])); agent = str(uuid.UUID(p['target']['agent_id']))
require(tenant == p['target']['tenant_id'] and agent == p['target']['agent_id'], 'canonical exact target')
scope = "tenant_id='" + tenant + "' AND agent_id='" + agent + "'"
query = ("SELECT json_build_object('jobs',(SELECT count(*) FROM jobs WHERE " + scope + "),"
"'outbox',(SELECT count(*) FROM outbox o JOIN jobs j USING(tenant_id,job_id,attempt_id) WHERE " + scope.replace('tenant_id=', 'j.tenant_id=').replace('agent_id=', 'j.agent_id=') + "),"
"'permits',(SELECT count(*) FROM execution_permits e JOIN jobs j USING(tenant_id,job_id,attempt_id) WHERE " + scope.replace('tenant_id=', 'j.tenant_id=').replace('agent_id=', 'j.agent_id=') + "),"
"'unresolved',(SELECT count(*) FROM jobs WHERE " + scope + " AND state NOT IN ('succeeded','failed','cancelled','expired')));")
root = Path('/opt/rust-discovery')
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
value = json.loads(subprocess.run(cmd, input=query.encode(), capture_output=True, check=True, timeout=20).stdout)
require(value == dict(jobs=30,outbox=30,permits=26,unresolved=0), 'same central closed count')
path = root/'devstate/root-policy/policy.json'; raw=path.read_bytes(); meta=path.lstat()
require(not path.is_symlink() and meta.st_uid==10001 and meta.st_mode&0o777==0o600
and hashlib.sha256(raw).hexdigest()==p['central_policy_sha256'], 'unchanged protected central policy')
require(dt.datetime.fromisoformat(json.loads(raw)['expires_at']) <= dt.datetime.now(dt.timezone.utc), 'Root scope remains closed')
return dict(counts=value,policy_sha256=p['central_policy_sha256'],closed=True,observed_at=dt.datetime.now(dt.timezone.utc).isoformat(),jobs_created=False,approvals=False)
def vm(p):
import base64
import fcntl
import hashlib
import json
import os
from pathlib import Path
import sqlite3
import stat
import subprocess
import traceback
import uuid
import datetime as dt
os.umask(0o077)
ident=str(uuid.UUID(p['id'])); require(ident==p['id'] and os.geteuid()==0,'fixed root installation case')
require(p['action'] in ('prepare','install','read') and p['type']=='rmm.script-console.preserving-install-v1','fixed operation')
parent=Path('/var/lib/rmm-bootstrap-history'); case=parent/('script-console-v1-'+ident)
intent={k:v for k,v in p.items() if k not in ('action','fresh_server_gate')}
def encode(v):return json.dumps(v,sort_keys=True,separators=(',',':')).encode()
def sha(b):return hashlib.sha256(b).hexdigest()
def fsync_dir(path):
fd=os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
try:os.fsync(fd)
finally:os.close(fd)
def create(path,raw,mode=0o600,gid=0):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as out:
out.write(raw);out.flush();os.fchmod(out.fileno(),mode);os.fchown(out.fileno(),0,gid);os.fsync(out.fileno())
fsync_dir(path.parent)
def safe(path,directory=False,owner=0):
m=path.lstat();require(not path.is_symlink() and m.st_uid==owner and not m.st_mode&0o022
and (stat.S_ISDIR(m.st_mode) if directory else stat.S_ISREG(m.st_mode) and m.st_nlink==1),'safe owned path')
return m
def run(args,**kwargs):return subprocess.run(args,capture_output=True,check=True,timeout=25,**kwargs).stdout
def svc(name):
return dict(line.split('=',1) for line in run(['systemctl','show',name,'-p','ActiveState','-p','MainPID','-p','UnitFileState']).decode().splitlines())
def marker(name, **fields):
create(case/(name+'.json'),encode(dict(id=ident,phase=name,**fields)))
def live_readback():
# This path never repairs, restarts, replaces or retries anything. A
# receipt is historical; only independently observed markers and native
# state describe what may have happened after an interrupted install.
started=dt.datetime.now(dt.timezone.utc).isoformat()
def observed_file(path, content=False):
if not os.path.lexists(path):return dict(present=False)
try:
m=path.lstat();value=dict(present=True,device=m.st_dev,inode=m.st_ino,
uid=m.st_uid,gid=m.st_gid,mode=stat.S_IMODE(m.st_mode),bytes=m.st_size,
symlink=stat.S_ISLNK(m.st_mode),regular=stat.S_ISREG(m.st_mode))
if not stat.S_ISREG(m.st_mode) or path.is_symlink() or m.st_size>256*1024*1024:
value['inspection_error']='unexpected_or_unbounded_file';return value
raw=path.read_bytes();value['sha256']=sha(raw)
if content:
require(m.st_uid==0 and not m.st_mode&0o022 and m.st_nlink==1
and m.st_size<8*1024*1024,'safe bounded historical receipt')
value['historical_content']=json.loads(raw)
return value
except Exception as e:return dict(present=True,inspection_error=type(e).__name__)
phase_names=('install-entered','stop-intent','stopped','agent-exchange-intent','agent-exchanged',
'helper-exchange-intent','helper-exchanged','full-files-installed','restart-intent',
'general-started','full-started','agent-started')
markers={name:observed_file(case/(name+'.json'),True) for name in phase_names}
services={}
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-full-maintenance-helper.service','rmm-apt-helper.service'):
try:
reply=subprocess.run(['systemctl','show',name,'-p','LoadState','-p','ActiveState',
'-p','SubState','-p','MainPID','-p','UnitFileState'],capture_output=True,timeout=25)
value=dict(exit_code=reply.returncode,properties=dict(line.split('=',1)
for line in reply.stdout.decode().splitlines() if '=' in line))
# A missing named Full unit is an observable partial state,
# not a reason to discard the other independent observations.
if reply.returncode!=0 and not (name=='rmm-full-maintenance-helper.service'
and reply.returncode==1 and value['properties'].get('LoadState')=='not-found'
and value['properties'].get('ActiveState')=='inactive'
and value['properties'].get('MainPID')=='0'):
value['inspection_error']='systemctl_show_failed'
pid=int(value['properties'].get('MainPID','0'))
if pid>0:
try:
proc=Path('/proc',str(pid));before=(proc/'stat').read_text()
status=(proc/'status').read_text().splitlines()
image=sha((proc/'exe').read_bytes())
require((proc/'stat').read_text().rsplit(')',1)[1].split()[19]
==before.rsplit(')',1)[1].split()[19],'same observed process start')
value['process']=dict(pid=pid,start_ticks=before.rsplit(')',1)[1].split()[19],sha256=image,
uids=next(x.split()[1:] for x in status if x.startswith('Uid:')),
gids=next(x.split()[1:] for x in status if x.startswith('Gid:')))
except Exception as e:value['process']=dict(pid=pid,inspection_error=type(e).__name__)
services[name]=value
except Exception as e:services[name]=dict(inspection_error=type(e).__name__)
files={}
for key,path in (('agent',Path('/usr/local/libexec/rmm-agent-remote')),('helper',Path('/usr/local/libexec/rmm-privileged-helper'))):
files[key+'_current']=observed_file(path)
for generation in ('previous','next'):files[key+'_'+generation]=observed_file(case/(key+'-'+generation))
historical=observed_file(case/'report.json',True)
return dict(id=ident,readback_only=True,observed_from=started,
observed_until=dt.datetime.now(dt.timezone.utc).isoformat(),install_entered=markers['install-entered']['present'],
services_may_have_changed=markers['stop-intent']['present'],
stop_confirmed=markers['stopped']['present'],markers=markers,services=services,files=files,
historical_report=historical,repair_attempted=False,retry_allowed=False,
product_state_writes=False,rmm_jobs=False,approvals=False)
def retained_files():
# Never open private device keys. Retain all file identities, sizes,
# modes and change timestamps; SQL content has its own full row witness.
value={}
for root in (Path('/var/lib/rmm-agent-remote'),Path('/var/lib/rmm-helper')):
for path in [root,*sorted(root.rglob('*'))]:
m=path.lstat();require(not path.is_symlink() and (stat.S_ISDIR(m.st_mode) or stat.S_ISREG(m.st_mode)),'no unexpected retained state object')
if str(path).endswith(('-journal','-wal','-shm')):raise RuntimeError('Settled journal required')
value[str(path)]=[m.st_dev,m.st_ino,m.st_uid,m.st_gid,m.st_mode,m.st_size,m.st_mtime_ns,m.st_ctime_ns]
return value
def sql_witness():
result={}
for name,identity in p['preflight']['database_inodes'].items():
path=Path(name);m=path.lstat();require([m.st_dev,m.st_ino]==identity and not path.is_symlink(),'preserved DB identity')
require(all(not os.path.lexists(str(path)+suffix) for suffix in ('-wal','-shm','-journal')),'no live SQLite sidecars')
db=sqlite3.connect(path.as_uri()+'?mode=ro',uri=True)
try:
db.execute('PRAGMA query_only=ON');db.execute('BEGIN')
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','retained SQL integrity')
tables=dict(db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name"))
rows={}
for table in tables:
require(table.replace('_','').isalnum(),'fixed SQLite identifier')
rows[table]=[sha(encode([dict(blob_sha256=sha(v),bytes=len(v)) if isinstance(v,bytes) else v for v in row]))
for row in db.execute('SELECT rowid,* FROM "'+table+'" ORDER BY rowid')]
result[name]=dict(schema=tables,rows=rows,identity=identity)
if path.name=='journal.sqlite':
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]==0,'no running job')
if path.parent.name=='rmm-helper':
current=db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY grant_id').fetchall()
exact={r[0]:sha(json.dumps([dict(sha256=sha(v),size=len(v)) if isinstance(v,bytes) else v for v in r],separators=(',',':')).encode()) for r in current}
require(exact==p['preflight']['root_rows'] and len(exact)==18 and all(r[5] is not None for r in current),'exact retained Root18')
finally:db.close()
return result
safe(parent,True)
if p['action']=='prepare':
before=snapshot_vm(p['preflight'])
require(before['controls']==p['baseline']['controls'] and before['processes']==p['baseline']['processes']
and before['databases']==p['baseline']['databases'],'fresh original preflight still exact')
case.mkdir(mode=0o700);create(case/'intent.json',encode(intent))
return dict(id=ident,prepared=True,services_changed=False,product_state_writes=False)
safe(case,True);safe(case/'intent.json')
require((case/'intent.json').read_bytes()==encode(intent),'original immutable install intent')
if p['action']=='read':
return live_readback()
require(not os.path.lexists(case/'install-entered.json'),'same case install must never repeat')
gate=p['fresh_server_gate']
require(gate['closed'] is True and gate['counts']==dict(jobs=30,outbox=30,permits=26,unresolved=0)
and gate['policy_sha256']==p['central_policy_sha256'],'fresh separate closed server gate')
before=snapshot_vm(p['preflight'])
require(before['controls']==p['baseline']['controls'] and before['processes']==p['baseline']['processes']
and before['databases']==p['baseline']['databases'],'exact pre-stop CAS including PIDs/startticks')
binaries={'agent':Path('/usr/local/libexec/rmm-agent-remote'),'helper':Path('/usr/local/libexec/rmm-privileged-helper')}
for key,path in binaries.items():
upload=case/(key+'-next');safe(upload)
require(sha(upload.read_bytes())==p['artifacts'][key]['sha256'] and upload.stat().st_size==p['artifacts'][key]['bytes'],'exact native-qualified upload')
marker('install-entered',install_once=True)
report=dict(id=ident,stage='entered',completed_preserving_bootstrap=False,signed_sync_verified=False,
services_changed=False,services_may_have_changed=False,root_consumed=18,root_unresolved=0,rmm_jobs=False,approvals=False,
package_effects=False,reboot_requested=False,apt_endpoint_enabled=False,apt_current_runtime_qualified=False,
full_job_accepted=False)
def save():
pending=case/'report-next';create(pending,encode(report));os.replace(pending,case/'report.json');fsync_dir(case)
locks=[]
try:
save();marker('stop-intent',services_may_have_changed=True)
report.update(stage='stop_requested',services_may_have_changed=True);save()
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service'])
require(all(svc(n)['ActiveState']=='inactive' for n in ('rmm-agent-remote.service','rmm-helper-cloud.service')),'both old daemons stopped')
marker('stopped',confirmed=True);report['services_changed']=True;save()
for path in (Path('/var/lib/rmm-agent-remote/agent.lock'),Path('/var/lib/rmm-helper/executor.lock')):
fd=os.open(path,os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB);locks.append(fd)
frozen_sql=sql_witness();frozen_files=retained_files()
create(case/'sql-before.json',encode(frozen_sql));create(case/'files-before.json',encode(frozen_files))
for key,path in binaries.items():
safe(path);require(sha(path.read_bytes())==p['preflight']['old_artifacts'][key],'unchanged old binary immediately before exchange')
require(path.stat().st_dev==case.stat().st_dev,'same-filesystem preserving exchange')
marker(key+'-exchange-intent',may_have_changed=True)
os.rename(path,case/(key+'-previous'));fsync_dir(path.parent);fsync_dir(case)
os.chmod(case/(key+'-next'),0o755);os.rename(case/(key+'-next'),path);fsync_dir(path.parent);fsync_dir(case)
marker(key+'-exchanged',confirmed=True)
report['stage']=key+'_exchanged';save()
helper=binaries['helper'];env={'PATH':'/usr/sbin:/usr/bin:/sbin:/bin','LANG':'C','LC_ALL':'C','RMM_HELPER_CONFIG':'/etc/rmm-remote/helper-general.json'}
general=json.loads(run([str(helper),'--inspect-general-profile'],env=env))
require(general==p['general_pin'],'new General generation retains identical public policy pin')
for path,expected in p['baseline']['controls'].items():
if path in [str(x) for x in binaries.values()]:continue
item=Path(path);m=item.lstat()
actual=dict(device=m.st_dev,inode=m.st_ino,mode=stat.S_IMODE(m.st_mode),uid=m.st_uid,gid=m.st_gid,bytes=m.st_size,sha256=sha(item.read_bytes()))
require(not item.is_symlink() and actual==expected,'General/APT/package control bytes/inodes/modes unchanged')
full=p['baseline']['full_config_candidate'];raw=base64.b64decode(full['bytes_b64'],validate=True)
require(sha(raw)==full['sha256'] and json.loads(raw)['helper_binary_sha256']==p['artifacts']['helper']['sha256'],'exact reviewed Fullconfig')
config=Path('/etc/rmm-remote/helper-full-maintenance.json');create(config,raw,0o600)
env['RMM_HELPER_CONFIG']=str(config)
pin=json.loads(run([str(helper),'--inspect-full-maintenance-profile'],env=env))
require(pin['schema_version']==2 and pin['profile']=='full_maintenance_v1' and pin['identity']==general['identity']
and pin['helper_binary_sha256']==p['artifacts']['helper']['sha256'],'actual new image/profile/identity proof')
pinpath=Path('/etc/rmm-remote/full-maintenance-helper-proof.json');create(pinpath,json.dumps(pin,separators=(',',':')).encode(),0o640,984)
unit=Path('/etc/systemd/system/rmm-full-maintenance-helper.service');create(unit,base64.b64decode(p['unit_bytes_b64'],validate=True),0o644)
dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/60-full-maintenance.conf')
safe(dropin.parent,True);create(dropin,b'[Service]\nEnvironment=RMM_FULL_MAINTENANCE_HELPER_SOCKET=/run/rmm-helper-maintenance/control.sock\nEnvironment=RMM_FULL_MAINTENANCE_POLICY_FILE=/etc/rmm-remote/full-maintenance-helper-proof.json\n',0o644)
require(sql_witness()==frozen_sql and retained_files()==frozen_files,'all preexisting SQL rows/bytes/inodes/keys unchanged before restart')
marker('full-files-installed',preserved_before_restart=True)
report.update(stage='installed_preserved',general_pin=general,full_pin=pin,preserved_before_restart=True);save()
for fd in reversed(locks):fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
locks.clear();run(['systemctl','daemon-reload']);run(['systemd-analyze','verify',str(unit)])
names=('User','Group','NoNewPrivileges','PrivateNetwork','PrivateTmp','ProtectSystem','ProtectHome',
'ProtectKernelTunables','ProtectKernelModules','ProtectControlGroups','RestrictSUIDSGID',
'RestrictAddressFamilies','SystemCallFilter','RootDirectory','RootImage')
full_unit=dict(line.split('=',1) for line in run(['systemctl','show','rmm-full-maintenance-helper.service',
*[argument for name in names for argument in ('-p',name)]]).decode().splitlines())
require(full_unit['User']=='root' and full_unit['Group']=='rmm-agent'
and all(full_unit[name]=='no' for name in names[2:11])
and all(full_unit[name]=='' for name in names[11:]),'effective Full unit must not inherit General sandbox')
report['effective_full_unit']=full_unit;save()
marker('restart-intent',services_may_have_changed=True)
for name,label in (('rmm-helper-cloud.service','general'),('rmm-full-maintenance-helper.service','full'),('rmm-agent-remote.service','agent')):
run(['systemctl','start',name]);require(svc(name)['ActiveState']=='active','new exact service active')
marker(label+'-started',confirmed=True)
processes={}
for name,key,uid in (('rmm-helper-cloud.service','general',0),('rmm-full-maintenance-helper.service','full',0),('rmm-agent-remote.service','agent',993)):
pid=int(svc(name)['MainPID']);status=Path('/proc',str(pid),'status').read_text().splitlines()
uids=next(x.split()[1:] for x in status if x.startswith('Uid:'));gids=next(x.split()[1:] for x in status if x.startswith('Gid:'))
image=sha(Path('/proc',str(pid),'exe').read_bytes());expected=p['artifacts']['agent' if key=='agent' else 'helper']['sha256']
require(uids==[str(uid)]*4 and gids==['984']*4 and image==expected,'actual new service accounts/code generation')
processes[key]=dict(pid=pid,uids=uids,gids=gids,sha256=image)
require(svc('rmm-apt-helper.service')['ActiveState']=='inactive' and svc('rmm-apt-helper.service')['UnitFileState']=='disabled','APT remains already closed')
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip()==p['preflight']['boot_id'],'no boot transition')
report.update(stage='new_generations_started',completed_preserving_bootstrap=True,processes=processes,boot_id=p['preflight']['boot_id']);save()
except Exception as e:
report.update(stage='incomplete_manual_review',error_class=type(e).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(e.__traceback__)]);save()
finally:
for fd in reversed(locks):fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
return report
def rpc(prefix, function, payload, folder):
code=inspect.getsource(require)+'\n'+inspect.getsource(pre.snapshot_vm)+'\n'+inspect.getsource(function)
code+='\ntry:\n print(__import__("json").dumps('+function.__name__+'('+repr(payload)+')))\nexcept Exception as e:\n print(__import__("json").dumps(dict(bootstrap_error=True,error_class=type(e).__name__)))\n'
return support.transport(prefix,'python3 -',code.encode(),folder,timeout=230)
def server_prefix():
cfg=support.access.load_config(pre.ACCESS_ROOT/'credentials/server.conf')
require(cfg['address']=='88.99.80.202' and cfg['username']=='root' and cfg.get('port','22')=='22' and cfg.get('ssh_key_path'),'fixed server bootstrap transport')
return ['ssh','-F','/dev/null','-T','-p','22','-o','StrictHostKeyChecking=yes','-o','BatchMode=yes','-o','IdentitiesOnly=yes','-o','ForwardAgent=no','-o','ConnectTimeout=10','-i',str(Path(cfg['ssh_key_path']).expanduser()),'root@88.99.80.202']
if __name__=='__main__':
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('mode',choices=('prepare-local','stage','install','read'))
parser.add_argument('--snapshot',type=Path);parser.add_argument('--snapshot-sha');parser.add_argument('--case');args=parser.parse_args()
os.umask(0o077);STATE.mkdir(mode=0o700,parents=True,exist_ok=True);support.private(STATE,True)
if args.mode=='prepare-local':
require(args.case is None and args.snapshot is not None and args.snapshot_sha,'fresh reviewed snapshot required')
baseline=support.load(args.snapshot);require(support.sha(args.snapshot)==args.snapshot_sha and baseline['fresh_target_cas'] is True
and baseline['install_authorized'] is False and baseline['root_consumed']==18 and baseline['root_unresolved']==0,'exact fresh read-only snapshot')
prepath=args.snapshot.parent/'intent.json';previous=support.load(prepath)
require(previous['runner_sha256']==PREFLIGHT_SHA and previous['manifest_sha256']==pre.native.MANIFEST_SHA,'reviewed preflight generation')
case=str(uuid.uuid4());folder=STATE/case;folder.mkdir(mode=0o700)
manifest=support.load(pre.native.PACK/'manifest.json');old=support.load(pre.PRIOR)
require(support.sha(pre.native.PACK/'manifest.json')==pre.native.MANIFEST_SHA
and support.sha(pre.PRIOR)==pre.PRIOR_SHA and support.sha(pre.NATIVE)==pre.NATIVE_SHA
and all(support.sha(ROOT/n)==h for n,h in manifest['source_files'].items()),'actual frozen source/native provenance')
intent=dict(type='rmm.script-console.preserving-install-v1',id=case,runner_sha256=support.sha(__file__),preflight_source_sha256=PREFLIGHT_SHA,
manifest_sha256=pre.native.MANIFEST_SHA,snapshot_sha256=args.snapshot_sha,snapshot_path=str(args.snapshot.absolute()),preflight=previous,baseline=baseline,
target=previous['target'],central_policy_sha256=old['central_policy_sha256'],general_pin=old['general_pin'],
artifacts={k:{field:manifest['artifacts'][name][field] for field in ('sha256','bytes')} for k,name in (('agent','rmm-agent'),('helper','rmm-privileged-helper'))},
unit_bytes_b64=base64.b64encode((ROOT/'deploy/linux/rmm-full-maintenance-helper.service').read_bytes()).decode())
support.save(folder/'intent.json',intent)
else:
require(args.case is not None and args.snapshot is None and args.snapshot_sha is None,'original case only')
case=str(uuid.UUID(args.case));require(case==args.case,'canonical case');folder=STATE/case;intent=support.load(folder/'intent.json')
require(intent['runner_sha256']==support.sha(__file__) and intent['preflight_source_sha256']==PREFLIGHT_SHA,'original same-source intent')
require(support.sha(pre.native.PACK/'manifest.json')==pre.native.MANIFEST_SHA
and all(support.sha(ROOT/n)==h for n,h in support.load(pre.native.PACK/'manifest.json')['source_files'].items()),'frozen158 inputs remain exact')
prefix=support.access_settings(pre.ACCESS_ROOT,'linux')
if args.mode=='stage':
support.save(folder/'stage-entered-local.json',dict(stage_once=True))
result=rpc(prefix,vm,dict(intent,action='prepare'),folder);support.save(folder/'prepared-remote.json',result)
require(result['prepared'] is True,'new exact remote case')
for key,name in (('agent','rmm-agent'),('helper','rmm-privileged-helper')):
support.upload(prefix,dict(path=str(pre.native.PACK/name),**intent['artifacts'][key]),REMOTE+case+'/'+key+'-next',folder,key)
elif args.mode=='install':
support.load(folder/'prepared-remote.json');support.save(folder/'install-entered-local.json',dict(install_once=True,no_retry=True))
gate=rpc(server_prefix(),server_gate,intent,folder);support.save(folder/'fresh-server-gate.json',gate)
result=rpc(prefix,vm,dict(intent,action='install',fresh_server_gate=gate),folder);support.save(folder/'install-reply.json',result)
require(result.get('completed_preserving_bootstrap') is True and result.get('signed_sync_verified') is False
and result.get('full_job_accepted') is False and result.get('rmm_jobs') is False and result.get('approvals') is False,
'installation incomplete; same-case readback/manual review only')
else:
result=rpc(prefix,vm,dict(intent,action='read'),folder);support.save(folder/('readback-'+str(uuid.uuid4())+'.json'),result)
print(json.dumps(dict(case_id=case,intent=str(folder/'intent.json'),intent_sha256=support.sha(folder/'intent.json'),mode=args.mode,
target_called=args.mode!='prepare-local',rmm_jobs=False,approvals=False)))
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env python3
"""Link current Windows runtime/units after the actual matching Linux84 proof.
No VM access, unit execution, enrollment, capability activation or deployment.
Windows native lists remain explicitly pending; old42 names are never relabelled.
"""
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import uuid
ROOT = Path(__file__).resolve().parents[1]
PACK = ROOT / 'devstate/script-console-qualification/run-29461892-343f-42f4-b469-88703535425e'
PACK_SHA = 'ddd1a78d1cd289b02b1f98572f9652171ec00be84e01ce099fd722d689bcaacd'
LINUX = ROOT / 'devstate/script-console-native/7b61ce46-7913-492b-9c6b-414a0c2b6ffc/passed-e214cd9b-21ac-48ce-82dc-d10364329d04.json'
LINUX_SHA = 'a831a70a3cf76a3e8a9bf5820dcca7100d2ef9860dd9388539fe9c13f949f470'
IMAGE = 'sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9'
PIPE_SHA = 'ce525dab23f2e4faa2c9816280c1a5a8b40e56d0e63a350256e83b2df1d9229d'
def sha(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def require(value, code):
if not value:
raise RuntimeError(code)
def load(path):
require(not path.is_symlink() and path.stat().st_uid == os.getuid()
and path.stat().st_mode & 0o777 == 0o600, 'private source evidence')
return json.loads(path.read_bytes())
def main():
os.umask(0o077)
require(sha(PACK / 'manifest.json') == PACK_SHA and sha(LINUX) == LINUX_SHA, 'exact current source and actual Linux84 proof')
source = load(PACK / 'manifest.json'); proof = load(LINUX)
require(proof['verified'] is True and proof['manifest_sha256'] == PACK_SHA
and proof['native']['passed'] == 84 and proof['native']['ignored_database_cases'] == 5
and proof['native']['all_lists_before_any_tests'] is True
and all(proof['native'][flag] is False for flag in ('product_state_writes','rmm_jobs','capabilities_activated','approvals')),
'actual bounded Linux-first proof, not an old fixture count')
frozen = source['source_files']
require(len(frozen) == 158 and all(sha(ROOT / name) == value for name, value in frozen.items())
and sha(ROOT / 'crates/agent-windows/src/helper_ipc.rs') == PIPE_SHA, 'unchanged current source including actual PipeV2')
state = ROOT / 'devstate/script-console-windows-link'; state.mkdir(mode=0o700, parents=True, exist_ok=True)
out = state / ('run-' + str(uuid.uuid4())); out.mkdir(mode=0o700)
target = ROOT / 'devstate/script-console-windows-target'; target.mkdir(mode=0o700, exist_ok=True)
checks = {}
def command(label, args):
value = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
log = out / (label + '.log'); log.write_bytes(value.stdout)
checks[label] = dict(exit_code=value.returncode, log_sha256=sha(log))
require(value.returncode == 0, 'link failed: inspect private ' + str(log))
return value.stdout
docker = ['docker','run','--rm','--network','none','--read-only','--tmpfs','/tmp:rw,size=512m',
'-v',str(ROOT)+':/src:ro','-v',str(target)+':/target','-e','CARGO_TARGET_DIR=/target','--entrypoint','cargo',IMAGE]
common = ['--manifest-path','/src/Cargo.toml','--locked','--offline','--target','x86_64-pc-windows-gnu',
'-p','rmm-agent-windows','--features','system-helper','--release']
units = command('windows-current-unit-link',docker+['test',*common,'--lib','--bin','rmm-agent-windows','--no-run','--message-format=json'])
binaries = command('windows-current-runtime-link',docker+['build',*common,'--bins','--message-format=json'])
artifacts = {}
for kind, raw in (('units', units), ('runtime', binaries)):
for line in raw.splitlines():
try:
value = json.loads(line)
except ValueError:
continue
if value.get('reason') != 'compiler-artifact' or not value.get('executable'):
continue
name = value['target']['name']
if kind == 'units':
label = {'rmm_agent_windows':'windows-lib-tests.exe','rmm-agent-windows':'windows-agent-tests.exe'}.get(name)
require(value.get('profile',{}).get('test') is True, 'test artifact only')
else:
label = {'rmm-helper-windows':'rmm-helper-windows.exe','rmm-agent-windows':'rmm-agent-windows.exe'}.get(name)
if label is None:
continue
require(label not in artifacts, 'exact unique PE artifact')
linked = target / Path(value['executable']).relative_to('/target')
original = out / (label + '.original'); delivery = out / label
shutil.copyfile(linked, original); shutil.copyfile(linked, delivery)
command(label+'-strip', ['docker','run','--rm','--network','none','--read-only','-v',str(out)+':/artifacts',
'--entrypoint','x86_64-w64-mingw32-strip',IMAGE,'--strip-debug','/artifacts/'+label])
data = delivery.read_bytes(); offset = int.from_bytes(data[60:64], 'little')
require(data[:2] == b'MZ' and data[offset:offset+6] == b'PE\0\0\x64\x86', 'actual Windows AMD64 PE')
require(sha(linked) == sha(original), 'original linked bytes retained')
artifacts[label] = dict(sha256=sha(delivery),bytes=delivery.stat().st_size,original_sha256=sha(original),
original_bytes=original.stat().st_size,debug_stripped_delivery=True,native_executed=False)
require(set(artifacts) == {'windows-lib-tests.exe','windows-agent-tests.exe','rmm-helper-windows.exe','rmm-agent-windows.exe'}, 'complete current runtime/unit set')
require(all(sha(ROOT/name) == digest for name,digest in frozen.items()) and sha(LINUX)==LINUX_SHA, 'same158 inputs and actual Linux proof before/after')
result = dict(type='rmm.script-console.windows-link-pack',schema_version=1,source_manifest_sha256=PACK_SHA,
source_files=frozen,source_before_after_equal=True,linux_native_proof_sha256=LINUX_SHA,linux_actual_passed=84,
linux_explicit_database_ignored=5,image_id=IMAGE,windows_target='x86_64-pc-windows-gnu',pipe_v2_sha256=PIPE_SHA,
checks=checks,artifacts=artifacts,windows_native_lists_executed=False,windows_native_tests_executed=False,
require_all_actual_native_lists_before_any_test=True,reuses_old_42_test_expectation=False,
clock_qualified=False,capabilities_activated=False,deployed=False,rmm_jobs=False,approvals=False)
path = out/'manifest.json'; path.write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(dict(manifest=str(path),sha256=sha(path),windows_native_executed=False,capabilities_activated=False)))
if __name__=='__main__':
main()