Bind SYSTEM bootstrap to exact runtime and original process exit
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-16 14:37:53 +01:00
parent 9e64ccabdb
commit 4394b1bd85
4 changed files with 377 additions and 9 deletions
+119 -7
View File
@@ -7,6 +7,33 @@ results are readback-only, never an automatic retry or rollback.
"""
def inspect_counts(payload):
import json
import subprocess
if payload != {'operation': 'windows_system_scope_counts_v1'}:
raise RuntimeError('SYSTEM_SCOPE_COUNTS_OPERATION')
sql = "BEGIN READ ONLY; SELECT coalesce(json_object_agg(state,n),'{}') FROM (SELECT state,count(*) n FROM jobs WHERE tenant_id='11111111-1111-4111-8111-111111111111' GROUP BY state) q; COMMIT;"
out = subprocess.run(['docker', 'exec', '-i', 'rust-discovery-server-postgres-1', 'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm'], input=sql.encode(), capture_output=True, check=True, timeout=15)
return dict(read_only=True, counts_by_state=json.loads(out.stdout), outputs_read=False)
def inspect_namespaces(payload):
import json
import subprocess
if payload != {'operation': 'windows_system_scope_namespaces_v1'}:
raise RuntimeError('SYSTEM_SCOPE_NAMESPACE_OPERATION')
rows = {}
for role in ('api', 'gateway'):
raw = subprocess.check_output(['docker', 'inspect', 'rust-discovery-server-' + role + '-1'], timeout=15)
c = json.loads(raw)[0]; labels = c['Config']['Labels']
args = ['docker', 'compose', '--project-name', labels['com.docker.compose.project'], '--project-directory', labels['com.docker.compose.project.working_dir'], '--env-file', '/opt/rust-discovery/devstate/.env']
for path in labels['com.docker.compose.project.config_files'].split(','): args += ['-f', path]
model = json.loads(subprocess.check_output(args + ['config', '--format', 'json'], timeout=20))['services'][role]
fields = ('ipc', 'pid', 'userns_mode', 'devices')
rows[role] = dict(actual={k: c['HostConfig'].get(k) for k in ('IpcMode', 'PidMode', 'UsernsMode', 'Devices', 'Binds')}, resolved={k: model.get(k) for k in fields}, mounts=c['Mounts'], resolved_volumes=model.get('volumes'))
return dict(read_only=True, roles=rows, environment_reported=False)
def activate(payload):
import datetime as dt
import fcntl
@@ -25,7 +52,7 @@ def activate(payload):
require(os.geteuid() == 0, 'SYSTEM_SCOPE_ROOT_BOOTSTRAP')
require(set(payload) == {'operation', 'case_id', 'expected_roles', 'memberships', 'policy'}, 'SYSTEM_SCOPE_INPUT')
require(payload['operation'] == 'windows_system_script_scope_activate_v1', 'SYSTEM_SCOPE_OPERATION')
require(payload['operation'] in ('windows_system_script_scope_activate_v1', 'windows_system_script_scope_spec_read_v1'), 'SYSTEM_SCOPE_OPERATION')
ident = str(uuid.UUID(payload['case_id']))
require(ident == payload['case_id'], 'SYSTEM_SCOPE_CASE')
tenant = '11111111-1111-4111-8111-111111111111'
@@ -51,12 +78,37 @@ def activate(payload):
require(set(roles) == {'api', 'gateway'}, 'SYSTEM_SCOPE_BOTH_ROLES')
compose_files = None
config_hashes = None
original_environments = {}
original_containers = {}
project_directory = None
# Same runtime projection as server_deploy_205.shape; retain the value in
# private RAM to compare the one intended env/mount delta after recreation.
def runtime_shape(c):
conf, host = c['Config'], c['HostConfig']
return dict(user=conf['User'], env=sorted(conf['Env']), entrypoint=conf['Entrypoint'], cmd=conf['Cmd'], working_dir=conf['WorkingDir'],
mounts=sorted(c['Mounts'], key=lambda m: m['Destination']), binds=sorted(host.get('Binds') or []), networks=sorted(c['NetworkSettings']['Networks']),
labels={k: v for k, v in (conf.get('Labels') or {}).items() if not k.startswith('com.docker.compose.')},
host={k: host.get(k) for k in ('ReadonlyRootfs', 'PortBindings', 'CapDrop', 'CapAdd', 'SecurityOpt', 'RestartPolicy', 'Privileged', 'NetworkMode', 'IpcMode', 'PidMode', 'Devices', 'UsernsMode')})
def assert_private_role(role, c):
conf, host = c['Config'], c['HostConfig']
port = {'api': '18443', 'gateway': '18444'}[role]
require(conf['User'] == '10001:10001' and host['PortBindings'] == {'18443/tcp': [{'HostIp': '127.0.0.1', 'HostPort': port}]} and host['ReadonlyRootfs'] and host['CapDrop'] == ['ALL'] and host['SecurityOpt'] == ['no-new-privileges:true'] and not host['Privileged'], 'SYSTEM_SCOPE_PRIVATE_UNPRIVILEGED_RUNTIME')
for role in ('api', 'gateway'):
expected = roles[role]
require(set(expected) == {'container_id', 'image', 'environment', 'compose_files_sha256'}, 'SYSTEM_SCOPE_ROLE_WITNESS')
require(re.fullmatch('[a-f0-9]{64}', expected['container_id']) and re.fullmatch('sha256:[a-f0-9]{64}', expected['image']), 'SYSTEM_SCOPE_CONTAINER_ID')
info = json.loads(run(['docker', 'inspect', expected['container_id']]))[0]
require(info['State']['Running'] and info['Image'] == expected['image'], 'SYSTEM_SCOPE_IMAGE_CAS')
assert_private_role(role, info)
original_containers[role] = info
all_environment = {}
for entry in info['Config']['Env']:
key, separator, value = entry.partition('=')
require(separator and key not in all_environment, 'SYSTEM_SCOPE_ACTUAL_ENVIRONMENT')
all_environment[key] = value
original_environments[role] = all_environment # private RAM only; never report credentials
selected = {}
names = {'RMM_PROFILE', 'RMM_DEV_AUTH', 'RMM_SERVER_ROLE', 'RMM_UI_ORIGIN', 'RMM_DEV_SYSTEM_ENABLED', 'RMM_DEV_SYSTEM_POLICY_FILE'}
for entry in info['Config']['Env']:
@@ -67,6 +119,10 @@ def activate(payload):
require(selected == expected['environment'] and selected == dict(RMM_PROFILE='development', RMM_DEV_AUTH='true', RMM_SERVER_ROLE=role, RMM_UI_ORIGIN='https://localhost:18443'), 'SYSTEM_SCOPE_EXISTING_PRIVATE_DEV_PROFILE')
labels = info['Config']['Labels']
require(labels['com.docker.compose.project'] == 'rust-discovery-server' and labels['com.docker.compose.service'] == role, 'SYSTEM_SCOPE_COMPOSE_ROLE')
directory = labels['com.docker.compose.project.working_dir']
require(Path(directory).is_absolute() and Path(directory).is_relative_to(root) and Path(directory).is_dir(), 'SYSTEM_SCOPE_ACTUAL_PROJECT_DIRECTORY')
if project_directory is None: project_directory = directory
require(directory == project_directory, 'SYSTEM_SCOPE_SAME_PROJECT_DIRECTORY')
files = labels['com.docker.compose.project.config_files'].split(',')
require(files and len(files) <= 12 and all(Path(p).is_absolute() and Path(p).is_relative_to(root) for p in files), 'SYSTEM_SCOPE_COMPOSE_PATHS')
hashes = {}
@@ -81,11 +137,54 @@ def activate(payload):
# Bind mounts are deliberately additive; no existing policy/TLS/data
# mount or published port is changed by the generated overlay.
require(not any(m['Destination'] == '/system-policy' for m in info['Mounts']), 'SYSTEM_SCOPE_PREVIOUS_SYSTEM_MOUNT')
compose = base[:]
compose = base + ['--project-name', 'rust-discovery-server', '--project-directory', project_directory]
for path in compose_files:
compose += ['-f', path]
for role in ('api', 'gateway'):
require(run(compose + ['ps', '-q', role]).decode().strip() == roles[role]['container_id'], 'SYSTEM_SCOPE_CURRENT_CONTAINER_CAS')
resolved_model = json.loads(run(compose + ['config', '--format', 'json']))
configured = resolved_model['services']
for role in ('api', 'gateway'):
service = configured[role]; current = original_containers[role]; host = current['HostConfig']; conf = current['Config']
image = json.loads(run(['docker', 'image', 'inspect', service['image']]))[0]
require(image['Id'] == roles[role]['image'], 'SYSTEM_SCOPE_NO_IMAGE_CHANGE')
defaults = image['Config']
environment = dict(entry.split('=', 1) for entry in (defaults.get('Env') or []))
require(all(value is not None for value in service.get('environment', {}).values()), 'SYSTEM_SCOPE_RESOLVED_ENVIRONMENT_VALUES')
environment.update({key: str(value) for key, value in service.get('environment', {}).items()})
require(environment == original_environments[role], 'SYSTEM_SCOPE_FULL_ENVIRONMENT_CAS')
require(service.get('user', defaults.get('User', '')) == conf['User'] and service.get('working_dir', defaults.get('WorkingDir', '')) == conf['WorkingDir'], 'SYSTEM_SCOPE_RESOLVED_USER_WORKDIR')
require((defaults.get('Entrypoint') if service.get('entrypoint') is None else service['entrypoint']) == conf['Entrypoint'] and (defaults.get('Cmd') if service.get('command') is None else service['command']) == conf['Cmd'], 'SYSTEM_SCOPE_RESOLVED_ENTRYPOINT_COMMAND')
ports = {}
for port in service.get('ports', []):
require(isinstance(port, dict) and port.get('host_ip') == '127.0.0.1' and port.get('protocol', 'tcp') == 'tcp', 'SYSTEM_SCOPE_RESOLVED_LOOPBACK_PORT')
ports.setdefault(str(port['target']) + '/tcp', []).append(dict(HostIp=port['host_ip'], HostPort=str(port['published'])))
require(ports == host['PortBindings'], 'SYSTEM_SCOPE_RESOLVED_PORT_BINDINGS')
mounts = []; binds = []
for volume in service.get('volumes', []):
require(volume['type'] == 'bind' and Path(volume['source']).is_absolute(), 'SYSTEM_SCOPE_EXISTING_BIND_ONLY')
options = volume.get('bind', {}); mode = 'ro' if volume.get('read_only', False) else 'rw'
require(not options.get('selinux'), 'SYSTEM_SCOPE_NO_NEW_SELINUX_MODE')
propagation = options.get('propagation', 'rprivate')
# Compose represents create_host_path short binds in HostConfig.Binds;
# the existing strict long time-source mount uses HostConfig.Mounts
# and Docker reports Mode="", while RW remains explicitly false.
legacy_bind = options.get('create_host_path', False)
mounts.append(dict(Type='bind', Source=volume['source'], Destination=volume['target'], Mode=mode if legacy_bind else '', RW=not volume.get('read_only', False), Propagation=propagation))
if legacy_bind:
binds.append(volume['source'] + ':' + volume['target'] + ':' + mode + (',' + propagation if 'propagation' in options else ''))
require(sorted(mounts, key=lambda m: m['Destination']) == sorted(current['Mounts'], key=lambda m: m['Destination']) and sorted(binds) == sorted(host.get('Binds') or []), 'SYSTEM_SCOPE_RESOLVED_MOUNTS_AND_BINDS')
networks = sorted(resolved_model['networks'][name]['name'] for name in service.get('networks', {}))
require(networks == sorted(current['NetworkSettings']['Networks']) and not service.get('network_mode') and host['NetworkMode'] in networks, 'SYSTEM_SCOPE_RESOLVED_NETWORKS')
restart = service.get('restart', 'no').split(':')
require(dict(Name=restart[0], MaximumRetryCount=int(restart[1]) if len(restart) == 2 else 0) == host['RestartPolicy'], 'SYSTEM_SCOPE_RESOLVED_RESTART_POLICY')
require(bool(service.get('read_only', False)) == host['ReadonlyRootfs'] and service.get('cap_drop') == host['CapDrop'] and (service.get('cap_add') or None) == (host['CapAdd'] or None) and service.get('security_opt') == host['SecurityOpt'] and bool(service.get('privileged', False)) == host['Privileged'], 'SYSTEM_SCOPE_RESOLVED_SANDBOX')
require(service.get('ipc', 'private') == host['IpcMode'] and service.get('pid', '') == host['PidMode'] and service.get('userns_mode', '') == host['UsernsMode'] and (service.get('devices') or []) == (host['Devices'] or []), 'SYSTEM_SCOPE_RESOLVED_NAMESPACES_DEVICES')
require(not any(service.get(key) for key in ('volumes_from', 'tmpfs', 'use_api_socket', 'gpus', 'device_cgroup_rules', 'credential_spec')), 'SYSTEM_SCOPE_NO_UNMODELLED_AUTHORITY')
labels = dict(defaults.get('Labels') or {}); labels.update(service.get('labels') or {})
require({k: v for k, v in labels.items() if not k.startswith('com.docker.compose.')} == runtime_shape(current)['labels'], 'SYSTEM_SCOPE_RESOLVED_NON_COMPOSE_LABELS')
if payload['operation'] == 'windows_system_script_scope_spec_read_v1':
return dict(read_only=True, runtime_matches_resolved_compose=True, project_directory=project_directory, roles={role: hashlib.sha256(enc(runtime_shape(c))).hexdigest() for role, c in original_containers.items()}, private_environment_reported=False, target_mutations=False)
policy = payload['policy']
require(set(policy) == {'schema_version', 'profile', 'expires_at', 'scopes'} and policy['schema_version'] == 3 and policy['profile'] == 'development' and len(policy['scopes']) == 1, 'SYSTEM_SCOPE_NARROW_POLICY')
now = dt.datetime.now(dt.timezone.utc)
@@ -105,11 +204,6 @@ def activate(payload):
require(observed['ready'] and observed['helper_ready'] and not observed['blocked'] and observed['fresh'] and observed['same_certificate'], 'SYSTEM_SCOPE_CURRENT_AUTHENTICATED_PAIR')
needed = {'windows.base/1', 'rmm.sync.v1', 'helper.system.v1', 'system:run_script', 'script.full_maintenance/1', 'helper.binary.sha256:' + scope['helper_binary_sha256'], 'helper.policy.sha256:' + scope['helper_policy_sha256']}
require(needed <= set(observed['capabilities']) and 'linux.base/1' not in observed['capabilities'], 'SYSTEM_SCOPE_ACTUAL_QUALIFIED_CAPABILITIES')
# Resolve only public image names; never build, pull or alter a deployed image.
configured = json.loads(run(compose + ['config', '--format', 'json']))['services']
for role in ('api', 'gateway'):
actual = run(['docker', 'image', 'inspect', configured[role]['image'], '--format', '{{.Id}}']).decode().strip()
require(actual == roles[role]['image'], 'SYSTEM_SCOPE_NO_IMAGE_CHANGE')
lock = os.open(root / 'devstate/.windows-system-scope.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
@@ -126,6 +220,15 @@ def activate(payload):
create(mounted / 'policy.json', raw, key.st_uid, key.st_gid)
overlay = {'services': {role: {'environment': {'RMM_DEV_SYSTEM_ENABLED': 'true', 'RMM_DEV_SYSTEM_POLICY_FILE': '/system-policy/policy.json'}, 'volumes': [str(mounted) + ':/system-policy:ro']} for role in ('api', 'gateway')}}
overlay_path = case / 'system-policy.compose.json'; create(overlay_path, enc(overlay))
new_model = json.loads(run(compose + ['-f', str(overlay_path), 'config', '--format', 'json']))
restored_model = json.loads(json.dumps(new_model))
for role in ('api', 'gateway'):
service = restored_model['services'][role]
require(service['environment'].pop('RMM_DEV_SYSTEM_ENABLED') == 'true' and service['environment'].pop('RMM_DEV_SYSTEM_POLICY_FILE') == '/system-policy/policy.json', 'SYSTEM_SCOPE_RESOLVED_TWO_ENV_ADDITIONS')
additions = [v for v in service['volumes'] if v['target'] == '/system-policy']
require(len(additions) == 1 and additions[0]['type'] == 'bind' and additions[0]['source'] == str(mounted) and additions[0]['read_only'] is True, 'SYSTEM_SCOPE_RESOLVED_ONE_POLICY_MOUNT')
service['volumes'].remove(additions[0])
require(restored_model == resolved_model, 'SYSTEM_SCOPE_ONLY_INTENDED_COMPOSE_DELTA')
sql = "BEGIN; LOCK TABLE memberships,jobs,outbox,execution_permits,authorizations IN SHARE ROW EXCLUSIVE MODE; DO $scope$ BEGIN IF EXISTS(SELECT 1 FROM jobs WHERE tenant_id='" + tenant + "' AND state NOT IN ('succeeded','failed','cancelled','expired')) THEN RAISE EXCEPTION 'SYSTEM_SCOPE_INFLIGHT_JOB'; END IF;"
for row in memberships:
permissions = 'ARRAY[' + ','.join("'" + p + "'" for p in row['permissions']) + ']::text[]'
@@ -142,8 +245,17 @@ def activate(payload):
cid = run(compose + ['-f', str(overlay_path), 'ps', '-q', role]).decode().strip()
info = json.loads(run(['docker', 'inspect', cid]))[0]
require(info['Image'] == roles[role]['image'] and info['State']['Running'], 'SYSTEM_SCOPE_ROLE_RUNNING')
next_environment = dict(entry.split('=', 1) for entry in info['Config']['Env'])
expected_environment = dict(original_environments[role], RMM_DEV_SYSTEM_ENABLED='true', RMM_DEV_SYSTEM_POLICY_FILE='/system-policy/policy.json')
require(next_environment == expected_environment, 'SYSTEM_SCOPE_ONLY_TWO_ENVIRONMENT_ADDITIONS')
mount = [m for m in info['Mounts'] if m['Destination'] == '/system-policy']
require(len(mount) == 1 and mount[0]['Type'] == 'bind' and not mount[0]['RW'] and mount[0]['Source'] == str(mounted), 'SYSTEM_SCOPE_READ_ONLY_MOUNT')
assert_private_role(role, info)
expected_shape = runtime_shape(original_containers[role])
expected_shape['env'] = sorted(key + '=' + value for key, value in expected_environment.items())
expected_shape['mounts'] = sorted(expected_shape['mounts'] + [dict(Type='bind', Source=str(mounted), Destination='/system-policy', Mode='ro', RW=False, Propagation='rprivate')], key=lambda m: m['Destination'])
expected_shape['binds'] = sorted(expected_shape['binds'] + [str(mounted) + ':/system-policy:ro'])
require(runtime_shape(info) == expected_shape, 'SYSTEM_SCOPE_EXACT_POST_RUNTIME_DELTA')
after[role] = dict(container_id=cid, image=info['Image'])
result = dict(schema_version=1, activated=True, case_id=ident, expires_at=policy['expires_at'], policy_sha256=hashlib.sha256(raw).hexdigest(), roles=after, jobs_created=False, approvals_created=False, system_target_job_executed=False, automatic_retry=False)
create(case / 'activated.json', enc(result))