Files
rust-discovery/deploy/server_time_source_bootstrap.py
T
2026-11-15 01:57:21 +01:00

848 lines
44 KiB
Python

#!/usr/bin/env python3
"""Isolated time-source sourcepack and finite operator phases; no SSH/deploy/key setup.
Host phases are deliberately NOT invoked by prepare. Run them only through the
operator's independently checked server transport. An uncertain phase is read,
never repeated. This first slice does not provision pins, signer, or services.
"""
import argparse
import hashlib
import io
import json
import os
from pathlib import Path
import re
import selectors
import signal
import stat
import struct
import subprocess
import tarfile
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
PROFILE = 'rmm.server-time-source.operator.v1'
CRATES = ('time-attestation', 'time-source-linux', 'time-broker-linux')
PACKAGES = tuple('rmm-' + name for name in CRATES)
BUILDER = 'rust:1.99.1-bookworm'
LOCAL_BUILDER = 'sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6'
RUNTIME = 'sha256:6f8383f7c762ac77ed1013c94eeafa2b76838c00aec358a375f5847645ea87ec'
HOST_ROOT = Path('/var/lib/rmm-time-source-bootstrap')
PROBE = b'''// Generated operator-only source-quality probe; no signer or caller context.
fn main() {
if std::env::args_os().len() != 1 { std::process::exit(2); }
let result = (|| -> anyhow::Result<()> {
let mut source = rmm_time_source_linux::NativeTimeSource::open()?;
let sample = source.sample()?;
println!("{}", serde_json::json!({
"profile": "rmm.native-source-quality.once.v1",
"clock_authority": false, "signed_attestation": false,
"captured_utc_ns": sample.captured_utc_ns(),
"server_error_ns": sample.server_error_ns(),
"source_sync_age_ns": sample.source_sync_age_ns(),
"server_boot_id": sample.server_boot_id(),
"captured_monotonic_ns": sample.captured_monotonic_ns(),
"sampled_boottime_ns": sample.sampled_boottime_ns(),
"kernel_state": sample.kernel_state()
}));
Ok(())
})();
if result.is_err() { eprintln!("TIME_SOURCE_OPERATOR_PROBE_DENIED"); std::process::exit(1); }
}
'''
class OperatorDenied(RuntimeError):
"""Only explicit operator guards may produce a public failure reason."""
def require(ok, reason):
if not ok:
raise OperatorDenied(reason)
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def encoded(value):
return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode()
def identifier(value):
require(str(uuid.UUID(value)) == value and uuid.UUID(value).version == 4, 'canonical UUIDv4 case')
return value
def exclusive(path, raw, mode=0o600):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw)
stream.flush()
os.fsync(stream.fileno())
fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(fd)
finally:
os.close(fd)
def regular(path, limit=16 * 1024 * 1024, single_link=True):
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(fd, 'rb') as stream:
before = os.fstat(stream.fileno())
require(stat.S_ISREG(before.st_mode) and (not single_link or before.st_nlink == 1) and before.st_size <= limit,
'bounded single-link regular file')
raw = stream.read(limit + 1)
after = os.fstat(stream.fileno())
stable = lambda item: (item.st_dev, item.st_ino, item.st_uid, item.st_gid, item.st_mode,
item.st_nlink, item.st_size, item.st_mtime_ns, item.st_ctime_ns)
require(len(raw) <= limit and stable(before) == stable(after), 'stable bounded input')
return raw
def run(args, timeout=15, limit=1024 * 1024):
# Fixed commands only; no shell, environment/configuration interpolation.
result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, timeout=timeout, check=False)
require(len(result.stdout) <= limit and len(result.stderr) <= limit, 'bounded command output')
require(result.returncode == 0, 'fixed command failed')
return result.stdout
def package_identity(package):
return tuple(package.get(key) for key in ('name', 'version', 'source', 'checksum'))
def lock_package(block):
"""Fail-closed reader for Cargo's canonical lock-v4 package blocks, not TOML in general."""
value, dependencies, inside = {}, [], False
for line in block.strip().splitlines():
if inside:
if line == ']':
inside = False
else:
require(line.startswith(' "') and line.endswith('",'), 'canonical lock dependency line')
dependencies.append(json.loads(line.strip()[:-1]))
elif line == 'dependencies = [':
require('dependencies' not in value, 'unique dependency array')
value['dependencies'] = dependencies
inside = True
else:
match = re.fullmatch(r'(name|version|source|checksum) = ("[^"\\]*")', line)
require(match and match[1] not in value, 'canonical unique lock package field')
value[match[1]] = json.loads(match[2])
require(not inside and 'name' in value and 'version' in value, 'complete lock package')
if 'source' in value:
require(value['source'] == 'registry+https://github.com/rust-lang/crates.io-index' and
re.fullmatch('[0-9a-f]{64}', value.get('checksum', '')), 'locked crates.io checksum only')
return value
def subset_lock(raw):
"""Preserve exact original lock blocks, selecting their dependency closure."""
blocks = raw.decode().split('[[package]]')
require(blocks[0] == '# This file is automatically @generated by Cargo.\n'
'# It is not intended for manual editing.\nversion = 4\n\n', 'canonical Cargo lock v4 header')
packages = [lock_package(block) for block in blocks[1:]]
by_name = {}
for item in packages:
by_name.setdefault(item['name'], []).append(item)
held = {}
def visit(item):
identity = package_identity(item)
if identity in held:
return
held[identity] = item
for dependency in item.get('dependencies', []):
parts = dependency.split(' ')
matches = by_name[parts[0]]
if len(parts) > 1:
matches = [value for value in matches if value['version'] == parts[1]]
if len(parts) > 2:
matches = [value for value in matches if '(' + value.get('source', '') + ')' == parts[2]]
require(len(matches) == 1, 'unambiguous locked dependency')
visit(matches[0])
for name in PACKAGES:
matches = by_name[name]
require(len(matches) == 1 and 'source' not in matches[0], 'exact workspace package')
visit(matches[0])
selected = [block for block in blocks[1:]
if package_identity(lock_package(block)) in held]
output = (blocks[0] + ''.join('[[package]]' + block for block in selected)).encode()
require({package_identity(lock_package(p)) for p in output.decode().split('[[package]]')[1:]} == set(held),
'exact lock closure, no version/checksum changes')
return output
def workspace_recipe(root_raw):
"""Reuse original workspace values verbatim; only members/dependency set shrink."""
deps = ('anyhow', 'hex', 'rand', 'serde', 'serde_json', 'sha2', 'uuid')
lines = root_raw.decode().splitlines()
start = lines.index('[workspace.package]')
end = lines.index('[workspace.dependencies]')
package = '\n'.join(lines[start:end]).strip()
dependency_lines = []
for name in deps:
selected = [line for line in lines[end + 1:] if line.startswith(name + ' = ')]
require(len(selected) == 1, 'single original workspace dependency')
dependency_lines.extend(selected)
recipe = ('[workspace]\nresolver = "2"\nmembers = ' +
json.dumps(['crates/' + name for name in CRATES]) + '\n\n' + package +
'\n\n[workspace.dependencies]\n' + '\n'.join(dependency_lines) + '\n').encode()
return recipe
def compare_lock(original, generated):
original_blocks = original.decode().split('[[package]]')
generated_blocks = generated.decode().split('[[package]]')
require(original_blocks[0] == generated_blocks[0], 'same canonical lock version')
known = {package_identity(lock_package(block)) for block in original_blocks[1:]}
selected = [lock_package(block) for block in generated_blocks[1:]]
require(len(selected) == len({package_identity(item) for item in selected}) and
all(package_identity(item) in known for item in selected),
'every generated package version/source/checksum exists in public original lock')
require({item['name'] for item in selected if 'source' not in item} == set(PACKAGES),
'only three original workspace packages')
return len(selected)
def generate_lock(case, payload, original):
workspace = case / 'lock-workspace'
workspace.mkdir(mode=0o700)
for name, raw in payload.items():
path = workspace / name
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
exclusive(path, raw)
args = ['docker', 'run', '--rm', '--pull=never', '--network=none', '--read-only',
'--tmpfs=/tmp:rw,nosuid,nodev,size=64m', '--cap-drop=ALL',
'--security-opt=no-new-privileges', '--user', f'{os.getuid()}:{os.getgid()}',
'-v', str(workspace) + ':/src:rw', '--entrypoint=cargo', LOCAL_BUILDER,
'generate-lockfile', '--manifest-path', '/src/Cargo.toml', '--offline']
result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, timeout=120, check=False)
require(len(result.stdout) <= 1024 * 1024, 'bounded local Cargo lock generation output')
exclusive(case / 'lock-generation.log', result.stdout)
require(result.returncode == 0, 'local offline lock generation failed; retained original case log')
os.chmod(workspace / 'Cargo.lock', 0o600)
generated = regular(workspace / 'Cargo.lock')
count = compare_lock(original, generated)
# Cargo was allowed to alter only the generated lock file in this private recipe.
for name, raw in payload.items():
if name != 'Cargo.lock':
require(regular(workspace / name) == raw, 'original copied source unchanged by lock generation')
return generated, {'builder_image': LOCAL_BUILDER, 'offline': True,
'package_count': count, 'log_sha256': digest(result.stdout)}
def prepare(commit):
require(re.fullmatch('[0-9a-f]{40}', commit) is not None, 'full public source commit')
public = run(['git', '-C', str(ROOT), 'for-each-ref', '--format=%(objectname)', 'refs/remotes/origin/']).decode().splitlines()
require(commit in public, 'checkpoint must be an already fetched origin branch tip')
names = run(['git', '-C', str(ROOT), 'ls-tree', '-r', '--name-only', commit]).decode().splitlines()
inputs = ['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml']
inputs += [name for name in names if any(name == f'crates/{crate}/Cargo.toml' or
(name.startswith(f'crates/{crate}/src/') and name.endswith('.rs')) for crate in CRATES)]
require(len(inputs) >= 12 and len(inputs) <= 24, 'bounded complete three-crate source set')
source = {}
for name in sorted(inputs):
raw = run(['git', '-C', str(ROOT), 'show', commit + ':' + name])
require(regular(ROOT / name) == raw, 'working input differs from public checkpoint: ' + name)
source[name] = raw
payload = dict(source)
payload['Cargo.toml'] = workspace_recipe(source['Cargo.toml'])
payload['Cargo.lock'] = subset_lock(source['Cargo.lock'])
payload['crates/time-source-linux/examples/operator_probe.rs'] = PROBE
state = ROOT / 'devstate/server-time-source-bootstrap'
state.mkdir(parents=True, mode=0o700, exist_ok=True)
require(state.stat().st_uid == os.getuid() and stat.S_IMODE(state.stat().st_mode) == 0o700,
'private local case root')
case = state / str(uuid.uuid4())
case.mkdir(mode=0o700)
payload['Cargo.lock'], lock_proof = generate_lock(case, payload, source['Cargo.lock'])
# Provenance originals are not compiled, but their exact bytes travel with the pack.
for name in ('Cargo.toml', 'Cargo.lock'):
payload['provenance/' + name] = source[name]
manifest = {'profile': PROFILE, 'public_commit': commit, 'builder': BUILDER,
'operator_sha256': digest(Path(__file__).read_bytes()),
'lock_generation': lock_proof,
'source_files': [{'path': name, 'bytes': len(raw), 'sha256': digest(raw)}
for name, raw in sorted(source.items())],
'package_files': [{'path': name, 'bytes': len(raw), 'sha256': digest(raw)}
for name, raw in sorted(payload.items())],
'probe_only': True, 'provisioned': False}
payload['manifest.json'] = encoded(manifest)
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode='w', format=tarfile.USTAR_FORMAT) as archive:
for name, raw in sorted(payload.items()):
item = tarfile.TarInfo(name)
item.size, item.mode, item.uid, item.gid, item.mtime = len(raw), 0o600, 0, 0, 0
archive.addfile(item, io.BytesIO(raw))
exclusive(case / 'source.tar', buffer.getvalue())
exclusive(case / 'operator.py', Path(__file__).read_bytes())
exclusive(case / 'manifest.json', encoded(manifest))
receipt = {'case': case.name, 'source_tar_sha256': digest(buffer.getvalue()),
'source_tar_bytes': len(buffer.getvalue()), 'manifest_sha256': digest(encoded(manifest)),
'compiled': False, 'target_executed': False}
exclusive(case / 'prepared.json', encoded(receipt))
print(json.dumps({'path': str(case), **receipt}, sort_keys=True))
def protected_dir(path, create=False):
require(path.is_absolute(), 'absolute fixed operator path')
current = Path('/')
for part in path.parts[1:]:
current /= part
if create and not current.exists():
current.mkdir(mode=0o700)
info = current.lstat()
require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and info.st_mode & 0o022 == 0,
'root-owned protected directory ancestry')
return path
def host_case(value, create=False):
require(os.geteuid() == 0 and os.uname().sysname == 'Linux' and os.uname().machine == 'aarch64',
'root Linux ARM64 server only')
protected_dir(HOST_ROOT, create=create)
path = HOST_ROOT / identifier(value)
if create:
path.mkdir(mode=0o700)
protected_dir(path)
require(stat.S_IMODE(path.stat().st_mode) == 0o700, 'private operator case')
return path
def metadata(path):
item = path.lstat()
return {key: getattr(item, 'st_' + key) for key in ('dev', 'ino', 'uid', 'gid', 'mode', 'size', 'mtime_ns', 'ctime_ns')}
def runtime_witness():
names = ('gateway', 'api', 'postgres')
result = {}
for name in names:
container = 'rust-discovery-server-' + name + '-1'
# Never inspect Env, command arguments, health output, mounts or credentials.
raw = run(['docker', 'inspect', '--format',
'{{json .Image}} {{json .Config.User}} {{json .State.Running}} {{json .State.Pid}} {{json .State.StartedAt}} {{json .RestartCount}} {{json .NetworkSettings.Ports}}', container])
# Multiple JSON values are deliberately decoded separately (spaces inside quoted dates/users are safe).
decoder, values, text = json.JSONDecoder(), [], raw.decode().strip()
while text:
value, end = decoder.raw_decode(text)
values.append(value)
text = text[end:].lstrip()
require(len(values) == 7 and values[2] is True and values[3] > 1, 'live expected runtime container')
if name != 'postgres':
require(values[0] == RUNTIME and values[1] == '10001:10001', 'fixed runtime image and user')
for bindings in (values[6] or {}).values():
require(bindings is None or (name != 'postgres' and all(x['HostIp'] == '127.0.0.1' for x in bindings)),
'no public Docker port or PostgreSQL publication')
result[name] = values
return result
def fd_metadata(fd):
item = os.fstat(fd)
return {key: getattr(item, 'st_' + key) for key in
('dev', 'ino', 'uid', 'gid', 'mode', 'nlink', 'size', 'mtime_ns', 'ctime_ns')}
def held_bytes(fd, limit):
before = fd_metadata(fd)
require(stat.S_ISREG(before['mode']) and before['size'] <= limit, 'bounded held regular input')
raw = os.pread(fd, limit + 1, 0)
require(len(raw) <= limit and fd_metadata(fd) == before, 'stable held input')
return raw
def chrony_unit():
fields = ('MainPID', 'ControlPID', 'ActiveState', 'SubState', 'Type', 'ControlGroup', 'InvocationID', 'Job')
raw = run(['systemctl', 'show', '--all', '--property=' + ','.join(fields), '--', 'chrony.service'], limit=4096)
values = {}
for line in raw.decode('ascii').splitlines():
key, separator, value = line.partition('=')
require(separator and key in fields and key not in values, 'complete fixed Chrony unit properties')
values[key] = value
require(set(values) == set(fields) and values['MainPID'].isdigit() and int(values['MainPID']) > 1 and
values['ControlPID'] == '0' and values['ActiveState'] == 'active' and values['SubState'] == 'running' and
values['Type'] == 'notify' and values['ControlGroup'] == '/system.slice/chrony.service' and
re.fullmatch('[0-9a-f]{32}', values['InvocationID']) and int(values['InvocationID'], 16) > 0 and
not values['Job'], 'actual fixed active notify Chrony service')
return values
def named_matches(path, held):
named = metadata(path)
return all(held.get(key) == value for key, value in named.items())
def cgroup_members(path):
raw = regular(path / 'cgroup.procs', 256)
require(re.fullmatch(rb'(?:[1-9][0-9]*\n){3}', raw) is not None, 'exact three Chrony cgroup processes')
members = sorted(int(value) for value in raw.splitlines())
require(len(set(members)) == 3 and all(value > 1 for value in members), 'unique Chrony cgroup processes')
# No silently ignored descendant cgroup can hide additional service members.
with os.scandir(path) as entries:
require(not any(entry.is_dir(follow_symlinks=False) for entry in entries),
'no descendant Chrony cgroup')
return members
class ChronyProcess:
def __init__(self, pid):
self.pid, self.fds = pid, []
try:
self.pidfd = os.pidfd_open(pid)
self.fds.append(self.pidfd)
self.proc = Path('/proc') / str(pid)
self.procfd = os.open(self.proc, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
self.fds.append(self.procfd)
# The fixed proc exe magic link is followed once, then held and
# compared with the current executable before and after hashing.
self.image = os.open('exe', os.O_RDONLY | os.O_CLOEXEC, dir_fd=self.procfd)
self.fds.append(self.image)
self.original = self.observe()
except Exception:
self.close()
raise
def close(self):
for fd in reversed(self.fds):
os.close(fd)
self.fds = []
def observe(self):
poll = __import__('select').poll()
poll.register(self.pidfd, __import__('select').POLLIN)
require(not poll.poll(0), 'held Chrony process alive')
status = regular(self.proc / 'status', 65536).decode('ascii')
ids = {}
for label in ('Uid', 'Gid'):
match = re.search('^' + label + r':\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$', status, re.MULTILINE)
require(match and len(set(match.groups())) == 1, 'fixed real effective process IDs')
ids[label.lower()] = int(match[1])
first = regular(self.proc / 'stat', 8192).decode('ascii').rsplit(')', 1)[1].split()
require(len(first) >= 20 and first[19].isdigit() and first[1].isdigit(), 'process birth and parent')
require(regular(self.proc / 'cgroup', 4096) == b'0::/system.slice/chrony.service\n',
'actual unified Chrony cgroup membership')
image = fd_metadata(self.image)
current = os.stat('exe', dir_fd=self.procfd)
require(stat.S_ISREG(image['mode']) and image['uid'] == 0 and image['mode'] & 0o022 == 0 and
(current.st_dev, current.st_ino) == (image['dev'], image['ino']), 'held protected actual process image')
image_hash = digest(held_bytes(self.image, 16 * 1024 * 1024))
after = os.stat('exe', dir_fd=self.procfd)
require((after.st_dev, after.st_ino) == (image['dev'], image['ino']) and
fd_metadata(self.image) == image and regular(self.proc / 'stat', 8192).decode('ascii').rsplit(')', 1)[1].split()[19] == first[19]
and not poll.poll(0), 'actual process image and birth unchanged')
return {'pid': self.pid, 'birth_ticks': int(first[19]), 'parent_pid': int(first[1]),
'uid': ids['uid'], 'gid': ids['gid'], 'image': image, 'image_sha256': image_hash}
def socket_fds(self, inode):
fd = os.open('fd', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=self.procfd)
try:
names = os.listdir(fd)
require(len(names) <= 128 and all(value.isdigit() for value in names), 'bounded process descriptor set')
return sorted(int(value) for value in names if os.readlink(value, dir_fd=fd) == 'socket:[' + str(inode) + ']')
finally:
os.close(fd)
def command_socket_inode():
raw = regular(Path('/proc/net/unix'), 1024 * 1024)
rows = [line.split() for line in raw.decode('ascii').splitlines()[1:]]
matches = [row for row in rows if len(row) == 8 and row[7] == '/run/chrony/chronyd.sock']
require(len(matches) == 1 and matches[0][4] == '0002' and matches[0][6].isdigit(),
'unique fixed Unix datagram command socket')
return int(matches[0][6])
def chrony_observation():
unit = chrony_unit()
cgroup = Path('/sys/fs/cgroup/system.slice/chrony.service')
protected_dir(cgroup)
group_identity = metadata(cgroup)
members = cgroup_members(cgroup)
processes = []
pidfile = directory = -1
try:
for pid in members:
processes.append(ChronyProcess(pid))
main = int(unit['MainPID'])
by_pid = {process.pid: process for process in processes}
require(main in by_pid, 'unit wrapper belongs to actual service cgroup')
# /run/chrony is daemon-owned: a root-owned pidfile alone is NOT
# sufficient namespace authority. Bind the actual command-socket FD too.
protected_dir(Path('/run'))
directory = os.open('/run/chrony', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
dir_identity = fd_metadata(directory)
require(stat.S_ISDIR(dir_identity['mode']) and dir_identity['uid'] == 989 and
dir_identity['gid'] == 989 and stat.S_IMODE(dir_identity['mode']) == 0o700,
'actual fixed protected Chrony socket directory')
pidfile = os.open('chronyd.pid', os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory)
pid_identity = fd_metadata(pidfile)
require(pid_identity['uid'] == 0 and pid_identity['gid'] == 0 and pid_identity['nlink'] == 1 and
stat.S_IMODE(pid_identity['mode']) == 0o644, 'root single link fixed Chrony pidfile')
pid_raw = held_bytes(pidfile, 32)
require(re.fullmatch(rb'[1-9][0-9]*\n', pid_raw) is not None, 'canonical actual Chrony pidfile PID')
daemon = int(pid_raw.strip())
require(daemon in by_pid and daemon != main, 'pidfile daemon belongs to service cgroup')
child = next(pid for pid in members if pid not in (main, daemon))
wrapper, actual, helper = (by_pid[pid].original for pid in (main, daemon, child))
require(wrapper['uid'] == wrapper['gid'] == 0 and actual['uid'] == actual['gid'] == 989 and
helper['uid'] == helper['gid'] == 989 and actual['parent_pid'] == main and helper['parent_pid'] == daemon,
'actual fixed wrapper daemon child identity chain')
fixed_images = {}
for role, path, observed in (('wrapper', '/usr/bin/dash', wrapper), ('daemon', '/usr/sbin/chronyd', actual)):
protected_dir(Path(path).parent)
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
try:
fixed = fd_metadata(fd)
require(fixed == observed['image'] and digest(held_bytes(fd, 16 * 1024 * 1024)) == observed['image_sha256']
and named_matches(Path(path), fixed),
'actual process equals protected fixed executable')
fixed_images[role] = fixed
finally:
os.close(fd)
require(helper['image'] == actual['image'] and helper['image_sha256'] == actual['image_sha256'],
'actual Chrony child shares protected daemon image')
socket_path = Path('/run/chrony/chronyd.sock')
socket_identity = metadata(socket_path)
require(stat.S_ISSOCK(socket_identity['mode']) and socket_identity['uid'] == socket_identity['gid'] == 989 and
stat.S_IMODE(socket_identity['mode']) == 0o755, 'actual fixed Chrony filesystem socket')
inode = command_socket_inode()
holders = {process.pid: process.socket_fds(inode) for process in processes}
require(bool(holders[daemon]) and all(not fds for pid, fds in holders.items() if pid != daemon),
'unique actual service command socket holder')
for process in processes:
require(process.observe() == process.original, 'service process tuple unchanged')
require(chrony_unit() == unit and cgroup_members(cgroup) == members and metadata(cgroup) == group_identity and
fd_metadata(directory) == dir_identity and named_matches(Path('/run/chrony'), dir_identity) and
held_bytes(pidfile, 32) == pid_raw and fd_metadata(pidfile) == pid_identity and
named_matches(Path('/run/chrony/chronyd.pid'), pid_identity) and
metadata(socket_path) == socket_identity and command_socket_inode() == inode and
named_matches(Path('/usr/bin/dash'), fixed_images['wrapper']) and
named_matches(Path('/usr/sbin/chronyd'), fixed_images['daemon']) and
{process.pid: process.socket_fds(inode) for process in processes} == holders,
'Chrony unit namespace process pidfile and socket unchanged')
return {'pin': {'schema_version': 1, 'daemon_pid': daemon, 'daemon_birth_ticks': actual['birth_ticks'],
'daemon_uid': actual['uid'], 'daemon_gid': actual['gid'], 'daemon_image_sha256': actual['image_sha256']},
'unit': unit, 'processes': [process.original for process in processes],
'cgroup_metadata': group_identity, 'pidfile_metadata': pid_identity, 'pidfile_sha256': digest(pid_raw),
'directory_metadata': dir_identity, 'socket_metadata': socket_identity,
'socket_kernel_inode': inode, 'service_socket_fds': holders, 'fixed_images': fixed_images}
finally:
if pidfile >= 0:
os.close(pidfile)
if directory >= 0:
os.close(directory)
for process in processes:
process.close()
def preflight():
engine = run(['docker', 'version', '--format', '{{json .Server.Version}}']).decode().strip()
image_values = json.loads(run(['docker', 'image', 'inspect', '--format',
'[{{json .Id}},{{json .Architecture}},{{json .Os}}]', BUILDER]))
image = dict(zip(('Id', 'Architecture', 'Os'), image_values, strict=True))
require(image['Architecture'] == 'arm64' and image['Os'] == 'linux', 'cached ARM64 builder required')
observed = chrony_observation()
boot = regular(Path('/proc/sys/kernel/random/boot_id'), 128).decode().strip()
require(str(uuid.UUID(boot)) == boot, 'kernel boot identity')
return {'docker_engine_version': json.loads(engine),
'builder': {key: image[key] for key in ('Id', 'Architecture', 'Os')},
'runtime': runtime_witness(), 'boot_id': boot,
'chrony_observation': observed['pin'], 'chrony_binding': observed,
'chrony_image_metadata': observed['fixed_images']['daemon'], 'pins_provisioned': False,
'quality_sampled': False, 'registry_cache_verified': False}
def unpack(case, expected):
require(re.fullmatch('[0-9a-f]{64}', expected) is not None, 'exact operator-approved source hash')
raw = regular(case / 'source.tar', 2 * 1024 * 1024)
require(digest(raw) == expected, 'exact uploaded source archive')
files = {}
with tarfile.open(fileobj=io.BytesIO(raw), mode='r:') as archive:
for member in archive.getmembers():
require(member.isfile() and member.name not in files and member.size <= 1024 * 1024
and not Path(member.name).is_absolute() and '..' not in Path(member.name).parts,
'bounded regular unique archive entries')
files[member.name] = archive.extractfile(member).read()
require(len(files) <= 30 and 'manifest.json' in files, 'bounded source archive manifest')
manifest = json.loads(files['manifest.json'])
require(manifest['profile'] == PROFILE and manifest['builder'] == BUILDER and
manifest['operator_sha256'] == digest(Path(__file__).read_bytes()), 'reviewed operator profile')
require(set(files) == {'manifest.json'} | {x['path'] for x in manifest['package_files']}, 'exact manifest file set')
originals = {x['path']: x for x in manifest['source_files']}
for item in manifest['package_files']:
require(len(files[item['path']]) == item['bytes'] and digest(files[item['path']]) == item['sha256'],
'every source package hash and length')
for name, item in originals.items():
actual = files['provenance/' + name] if name in ('Cargo.toml', 'Cargo.lock') else files[name]
require(len(actual) == item['bytes'] and digest(actual) == item['sha256'], 'unchanged original checkpoint input')
require(files['Cargo.toml'] == workspace_recipe(files['provenance/Cargo.toml']) and
compare_lock(files['provenance/Cargo.lock'], files['Cargo.lock']) == manifest['lock_generation']['package_count'] and
manifest['lock_generation']['builder_image'] == LOCAL_BUILDER and
manifest['lock_generation']['offline'] is True and
files['crates/time-source-linux/examples/operator_probe.rs'] == PROBE,
'exact generated recipe, lock closure, source-only probe')
destination = case / 'workspace'
destination.mkdir(mode=0o700)
for name, value in sorted(files.items()):
target = destination / name
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
exclusive(target, value)
return {'source_tar_sha256': expected, 'manifest_sha256': digest(files['manifest.json']),
'public_commit': manifest['public_commit'], 'package_files': manifest['package_files']}
def verify_workspace(case):
staged = json.loads(regular(case / 'stage' / 'result.json'))
for item in staged['package_files']:
raw = regular(case / 'workspace' / item['path'])
require(digest(raw) == item['sha256'] and len(raw) == item['bytes'], 'original staged source bytes')
return staged
def long_command(args, log, timeout, cancel=None):
fd = os.open(log, os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY, 0o600)
with os.fdopen(fd, 'wb') as stream:
process = subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
start_new_session=True)
deadline, written = time.monotonic() + timeout, 0
try:
with selectors.DefaultSelector() as selector:
selector.register(process.stdout, selectors.EVENT_READ)
while selector.get_map():
remaining = deadline - time.monotonic()
require(remaining > 0, 'bounded command timeout; read original case only')
events = selector.select(remaining)
require(events, 'bounded command timeout; read original case only')
raw = os.read(process.stdout.fileno(), 65536)
if not raw:
selector.unregister(process.stdout)
break
allowed = 8 * 1024 * 1024 - written
stream.write(raw[:allowed])
written += min(len(raw), allowed)
require(len(raw) <= allowed, 'bounded command output exceeded; read original case only')
remaining = deadline - time.monotonic()
require(remaining > 0, 'bounded command timeout; read original case only')
code = process.wait(timeout=remaining)
except BaseException:
# The leader is still our unreaped child: its process group cannot
# have been recycled to an unrelated process.
if process.returncode is None:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
if cancel is not None:
cancel()
raise
finally:
process.stdout.close()
stream.flush()
os.fsync(stream.fileno())
return code
def cancel_owned_builder(cidfile, image_id, case_id):
# No name-based/global docker kill. Missing/uncertain identity is reported;
# the independent in-container 850s timeout remains the final bound.
if not cidfile.exists():
raise RuntimeError('no retained container ID; read original case, internal timeout remains')
cid = regular(cidfile, 128).decode().strip()
require(re.fullmatch('[0-9a-f]{64}', cid) is not None, 'exact own Docker container ID')
inspected = subprocess.run(['docker', 'inspect', '--format',
'[{{json .Id}},{{json .Image}},{{json .Name}},{{json (index .Config.Labels "rmm.time-source.case")}},{{json .State.Running}}]', cid],
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
if inspected.returncode != 0:
# --rm may already have removed our completed container. No fallback.
return
require(len(inspected.stdout) <= 4096, 'bounded own container metadata')
identity = json.loads(inspected.stdout)
require(identity[:4] == [cid, image_id, '/rmm-time-source-build-' + case_id, case_id],
'exact retained own container/image/case identity before cancellation')
if identity[4] is True:
run(['docker', 'kill', '--signal=KILL', cid], timeout=10, limit=4096)
def elf(path):
raw = regular(path, 32 * 1024 * 1024)
require(raw[:6] == b'\x7fELF\x02\x01' and struct.unpack('<H', raw[18:20])[0] == 183,
'actual ELF64 little-endian AArch64 artifact')
return {'bytes': len(raw), 'sha256': digest(raw), 'elf_machine': 183}
def build(case, phase, fetch):
staged = verify_workspace(case)
original = json.loads(regular(case / 'preflight/result.json'))
before = preflight()
require(before['builder'] == original['builder'] and before['boot_id'] == original['boot_id'] and
original['builder']['Architecture'] == 'arm64' and original['builder']['Os'] == 'linux' and
re.fullmatch('sha256:[0-9a-f]{64}', original['builder']['Id']),
'original preflight immutable builder and boot; no silent tag reimage')
image_id = original['builder']['Id']
target = phase / 'target'
target.mkdir(mode=0o700)
cidfile = phase / 'container.id'
# All Cargo lock/cache writes belong to this fresh bounded home. Offline
# may read the cached image registry; explicit fetch gets an empty registry.
setup = 'set -eu; '
if not fetch:
setup += 'if test -d /usr/local/cargo/registry; then ln -s /usr/local/cargo/registry /cargo/registry; fi; '
setup += 'exec cargo "$@"'
args = ['docker', 'run', '--rm', '--pull=never', '--network', 'bridge' if fetch else 'none',
'--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--pids-limit=256',
'--memory=2g', '--cpus=2', '--tmpfs=/tmp:rw,nosuid,nodev,size=256m',
'--tmpfs=/cargo:rw,nosuid,nodev,size=512m,mode=0700', '-e', 'CARGO_HOME=/cargo',
'-v', str(case / 'workspace') + ':/src:ro', '-v', str(target) + ':/out:rw',
'--name=rmm-time-source-build-' + case.name,
'--cidfile=' + str(cidfile),
'--label=rmm.time-source.case=' + case.name,
'-e', 'CARGO_TARGET_DIR=/out', '--entrypoint=/usr/bin/timeout', image_id,
'--kill-after=3s', '850s', '/bin/sh', '-c', setup, 'rmm-fixed-cargo',
'build', '--manifest-path', '/src/Cargo.toml', '--locked', '--release',
'-p', 'rmm-time-broker-linux', '-p', 'rmm-time-source-linux', '--bins', '--examples']
if not fetch:
args.append('--offline')
code = long_command(args, phase / 'build.log', 900,
cancel=lambda: cancel_owned_builder(cidfile, image_id, case.name))
after = preflight()
require(before['runtime'] == after['runtime'] and before['boot_id'] == after['boot_id'],
'RMM runtime metadata and boot preserved')
verify_workspace(case)
result = {'exit_code': code, 'registry_fetch_explicit': fetch, 'builder': before['builder'],
'before': before, 'after': after, 'staged_manifest_sha256': staged['manifest_sha256'],
'build_log_sha256': digest(regular(phase / 'build.log', 8 * 1024 * 1024)),
'runtime_metadata_equal': True, 'native_source_quality_tested': False}
if code == 0:
# Cargo may hardlink release outputs to deps/. Copy exact immutable bytes
# into fresh single-link root-owned artifacts before any probe execution.
artifacts = phase / 'artifacts'
artifacts.mkdir(mode=0o700)
result['artifacts'] = {}
for name in ('rmm-time-broker-linux', 'examples/operator_probe'):
raw = regular(target / 'release' / name, 32 * 1024 * 1024, single_link=False)
artifact = artifacts / Path(name).name
exclusive(artifact, raw, mode=0o500)
result['artifacts'][name] = elf(artifact)
else:
result['blocked'] = 'BUILD_FAILED_READ_LOG; offline cache is not assumed; no automatic retry/fetch'
return result
def probe(case, phase, expected):
require(re.fullmatch('[0-9a-f]{64}', expected) is not None, 'operator-approved exact probe SHA')
built = json.loads(regular(case / 'build' / 'result.json'))
path = case / 'build/artifacts/operator_probe'
artifact = elf(path)
require(built['exit_code'] == 0 and artifact == built['artifacts']['examples/operator_probe']
and artifact['sha256'] == expected, 'qualified original ARM64 source-only artifact')
verify_workspace(case)
# A previous provisioning action is a separate reviewed operation. Never synthesize pins here.
protected_dir(Path('/etc/rmm-time-source'))
pin = Path('/etc/rmm-time-source/chrony-pin.json')
meta = metadata(pin)
require(stat.S_ISREG(meta['mode']) and meta['uid'] == 0 and stat.S_IMODE(meta['mode']) == 0o600,
'real protected Chrony pin required')
protected_dir(Path('/run/rmm-time-source/clients'))
before = preflight()
args = ['systemd-run', '--quiet', '--wait', '--pipe', '--collect',
'--unit=rmm-time-source-probe-' + case.name, '--property=Type=exec',
'--property=User=root', '--property=Group=' + str(before['chrony_observation']['daemon_gid']),
'--property=SupplementaryGroups=10001', '--property=NoNewPrivileges=yes',
'--property=CapabilityBoundingSet=CAP_SYS_PTRACE', '--property=AmbientCapabilities=',
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
'--property=PrivateTmp=yes', '--property=PrivateDevices=yes',
'--property=RestrictAddressFamilies=AF_UNIX', '--property=IPAddressDeny=any',
'--property=SystemCallFilter=~@debug ptrace process_vm_readv process_vm_writev',
'--property=RuntimeMaxSec=3s', '--property=ReadWritePaths=/run/rmm-time-source/clients', str(path)]
code = long_command(args, phase / 'probe.log', 10)
after = preflight()
require(before['runtime'] == after['runtime'] and before['boot_id'] == after['boot_id'],
'RMM runtime metadata and boot preserved')
raw = regular(phase / 'probe.log', 16384)
sample = None
if code == 0:
sample = json.loads(raw)
require(sample['profile'] == 'rmm.native-source-quality.once.v1' and
sample['clock_authority'] is False and sample['signed_attestation'] is False,
'source-only single sample, no clock authority')
return {'exit_code': code, 'artifact': artifact, 'before': before, 'after': after,
'sample': sample, 'probe_log_sha256': digest(raw), 'runtime_metadata_equal': True,
'signer_provisioned': False, 'broker_started': False, 'http_deployed': False}
def host(args):
case = host_case(args.case, create=args.command == 'host-preflight')
if args.command == 'host-read':
# No phase creation, Docker actions, signals or rerun. Read only original bounded case records.
print(json.dumps({name: {'result': json.loads(regular(case / name / 'result.json'))
if (case / name / 'result.json').exists() else None,
'failure': json.loads(regular(case / name / 'failure.json'))
if (case / name / 'failure.json').exists() else None,
'intent_present': (case / name / 'intent.json').exists()}
for name in ('preflight', 'stage', 'build', 'probe')}, sort_keys=True))
return
name = args.command.removeprefix('host-')
phase = case / name
phase.mkdir(mode=0o700) # An existing intent OR result never permits rerun.
exclusive(phase / 'intent.json', encoded({'profile': PROFILE, 'case': case.name, 'phase': name,
'operator_sha256': digest(Path(__file__).read_bytes()), 'started_unix_ns': time.time_ns()}))
try:
if name == 'preflight':
result = preflight()
elif name == 'stage':
result = unpack(case, args.sha256)
elif name == 'build':
result = build(case, phase, args.registry_fetch)
else:
result = probe(case, phase, args.sha256)
exclusive(phase / 'result.json', encoded(result))
print(json.dumps({'case': case.name, 'phase': name, 'result': result}, sort_keys=True))
except Exception as error:
# Fixed controlled messages only; never write exception repr, paths from secrets, nested errors or env.
exclusive(phase / 'failure.json', encoded({'failed': True, 'exception_type': type(error).__name__,
'failure_reason': str(error) if isinstance(error, OperatorDenied) else 'unexpected_operator_exception',
'read_original_case_only': True}))
print(json.dumps({'case': case.name, 'phase': name, 'failed': True,
'failure_reason': str(error) if isinstance(error, OperatorDenied) else 'unexpected_operator_exception',
'read_original_case_only': True}, sort_keys=True))
raise SystemExit(1)
def main():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest='command', required=True)
prepare_parser = commands.add_parser('prepare')
prepare_parser.add_argument('--commit', required=True)
for name in ('preflight', 'stage', 'build', 'probe', 'read'):
command = commands.add_parser('host-' + name)
command.add_argument('--case', required=True)
if name in ('stage', 'probe'):
command.add_argument('--sha256', required=True)
if name == 'build':
command.add_argument('--registry-fetch', action='store_true',
help='explicit operator-authorized crates.io fetch using locked checksums; never OS updates')
args = parser.parse_args()
if args.command == 'prepare':
prepare(args.commit)
else:
host(args)
if __name__ == '__main__':
try:
main()
except Exception:
# A failure writing the private receipt must not print nested exceptions.
print(json.dumps({'failed': True, 'failure_reason': 'unexpected_operator_exception',
'read_original_case_only': True}, sort_keys=True))
raise SystemExit(1)