Record real clock bootstrap failures and preserve isolated operators
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -0,0 +1,665 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Isolated time-source sourcepack and finite operator phases; no SSH/deploy/key setup.
|
||||
|
||||
Host phases are deliberately NOT invoked by prepare. Run them only through the
|
||||
operator's independently checked server transport. An uncertain phase is read,
|
||||
never repeated. This first slice does not provision pins, signer, or services.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import selectors
|
||||
import signal
|
||||
import stat
|
||||
import struct
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROFILE = 'rmm.server-time-source.operator.v1'
|
||||
CRATES = ('time-attestation', 'time-source-linux', 'time-broker-linux')
|
||||
PACKAGES = tuple('rmm-' + name for name in CRATES)
|
||||
BUILDER = 'rust:1.99.1-bookworm'
|
||||
LOCAL_BUILDER = 'sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6'
|
||||
RUNTIME = 'sha256:6f8383f7c762ac77ed1013c94eeafa2b76838c00aec358a375f5847645ea87ec'
|
||||
HOST_ROOT = Path('/var/lib/rmm-time-source-bootstrap')
|
||||
PROBE = b'''// Generated operator-only source-quality probe; no signer or caller context.
|
||||
fn main() {
|
||||
if std::env::args_os().len() != 1 { std::process::exit(2); }
|
||||
let result = (|| -> anyhow::Result<()> {
|
||||
let mut source = rmm_time_source_linux::NativeTimeSource::open()?;
|
||||
let sample = source.sample()?;
|
||||
println!("{}", serde_json::json!({
|
||||
"profile": "rmm.native-source-quality.once.v1",
|
||||
"clock_authority": false, "signed_attestation": false,
|
||||
"captured_utc_ns": sample.captured_utc_ns(),
|
||||
"server_error_ns": sample.server_error_ns(),
|
||||
"source_sync_age_ns": sample.source_sync_age_ns(),
|
||||
"server_boot_id": sample.server_boot_id(),
|
||||
"captured_monotonic_ns": sample.captured_monotonic_ns(),
|
||||
"sampled_boottime_ns": sample.sampled_boottime_ns(),
|
||||
"kernel_state": sample.kernel_state()
|
||||
}));
|
||||
Ok(())
|
||||
})();
|
||||
if result.is_err() { eprintln!("TIME_SOURCE_OPERATOR_PROBE_DENIED"); std::process::exit(1); }
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
def require(ok, reason):
|
||||
if not ok:
|
||||
raise RuntimeError(reason)
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def encoded(value):
|
||||
return (json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n').encode()
|
||||
|
||||
|
||||
def identifier(value):
|
||||
require(str(uuid.UUID(value)) == value and uuid.UUID(value).version == 4, 'canonical UUIDv4 case')
|
||||
return value
|
||||
|
||||
|
||||
def exclusive(path, raw, mode=0o600):
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(raw)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def regular(path, limit=16 * 1024 * 1024, single_link=True):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
with os.fdopen(fd, 'rb') as stream:
|
||||
before = os.fstat(stream.fileno())
|
||||
require(stat.S_ISREG(before.st_mode) and (not single_link or before.st_nlink == 1) and before.st_size <= limit,
|
||||
'bounded single-link regular file')
|
||||
raw = stream.read(limit + 1)
|
||||
after = os.fstat(stream.fileno())
|
||||
stable = lambda item: (item.st_dev, item.st_ino, item.st_uid, item.st_gid, item.st_mode,
|
||||
item.st_nlink, item.st_size, item.st_mtime_ns, item.st_ctime_ns)
|
||||
require(len(raw) <= limit and stable(before) == stable(after), 'stable bounded input')
|
||||
return raw
|
||||
|
||||
|
||||
def run(args, timeout=15, limit=1024 * 1024):
|
||||
# Fixed commands only; no shell, environment/configuration interpolation.
|
||||
result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, timeout=timeout, check=False)
|
||||
require(len(result.stdout) <= limit and len(result.stderr) <= limit, 'bounded command output')
|
||||
require(result.returncode == 0, 'fixed command failed: ' + args[0])
|
||||
return result.stdout
|
||||
|
||||
|
||||
def package_identity(package):
|
||||
return tuple(package.get(key) for key in ('name', 'version', 'source', 'checksum'))
|
||||
|
||||
|
||||
def lock_package(block):
|
||||
"""Fail-closed reader for Cargo's canonical lock-v4 package blocks, not TOML in general."""
|
||||
value, dependencies, inside = {}, [], False
|
||||
for line in block.strip().splitlines():
|
||||
if inside:
|
||||
if line == ']':
|
||||
inside = False
|
||||
else:
|
||||
require(line.startswith(' "') and line.endswith('",'), 'canonical lock dependency line')
|
||||
dependencies.append(json.loads(line.strip()[:-1]))
|
||||
elif line == 'dependencies = [':
|
||||
require('dependencies' not in value, 'unique dependency array')
|
||||
value['dependencies'] = dependencies
|
||||
inside = True
|
||||
else:
|
||||
match = re.fullmatch(r'(name|version|source|checksum) = ("[^"\\]*")', line)
|
||||
require(match and match[1] not in value, 'canonical unique lock package field')
|
||||
value[match[1]] = json.loads(match[2])
|
||||
require(not inside and 'name' in value and 'version' in value, 'complete lock package')
|
||||
if 'source' in value:
|
||||
require(value['source'] == 'registry+https://github.com/rust-lang/crates.io-index' and
|
||||
re.fullmatch('[0-9a-f]{64}', value.get('checksum', '')), 'locked crates.io checksum only')
|
||||
return value
|
||||
|
||||
|
||||
def subset_lock(raw):
|
||||
"""Preserve exact original lock blocks, selecting their dependency closure."""
|
||||
blocks = raw.decode().split('[[package]]')
|
||||
require(blocks[0] == '# This file is automatically @generated by Cargo.\n'
|
||||
'# It is not intended for manual editing.\nversion = 4\n\n', 'canonical Cargo lock v4 header')
|
||||
packages = [lock_package(block) for block in blocks[1:]]
|
||||
by_name = {}
|
||||
for item in packages:
|
||||
by_name.setdefault(item['name'], []).append(item)
|
||||
held = {}
|
||||
|
||||
def visit(item):
|
||||
identity = package_identity(item)
|
||||
if identity in held:
|
||||
return
|
||||
held[identity] = item
|
||||
for dependency in item.get('dependencies', []):
|
||||
parts = dependency.split(' ')
|
||||
matches = by_name[parts[0]]
|
||||
if len(parts) > 1:
|
||||
matches = [value for value in matches if value['version'] == parts[1]]
|
||||
if len(parts) > 2:
|
||||
matches = [value for value in matches if '(' + value.get('source', '') + ')' == parts[2]]
|
||||
require(len(matches) == 1, 'unambiguous locked dependency')
|
||||
visit(matches[0])
|
||||
|
||||
for name in PACKAGES:
|
||||
matches = by_name[name]
|
||||
require(len(matches) == 1 and 'source' not in matches[0], 'exact workspace package')
|
||||
visit(matches[0])
|
||||
selected = [block for block in blocks[1:]
|
||||
if package_identity(lock_package(block)) in held]
|
||||
output = (blocks[0] + ''.join('[[package]]' + block for block in selected)).encode()
|
||||
require({package_identity(lock_package(p)) for p in output.decode().split('[[package]]')[1:]} == set(held),
|
||||
'exact lock closure, no version/checksum changes')
|
||||
return output
|
||||
|
||||
|
||||
def workspace_recipe(root_raw):
|
||||
"""Reuse original workspace values verbatim; only members/dependency set shrink."""
|
||||
deps = ('anyhow', 'hex', 'rand', 'serde', 'serde_json', 'sha2', 'uuid')
|
||||
lines = root_raw.decode().splitlines()
|
||||
start = lines.index('[workspace.package]')
|
||||
end = lines.index('[workspace.dependencies]')
|
||||
package = '\n'.join(lines[start:end]).strip()
|
||||
dependency_lines = []
|
||||
for name in deps:
|
||||
selected = [line for line in lines[end + 1:] if line.startswith(name + ' = ')]
|
||||
require(len(selected) == 1, 'single original workspace dependency')
|
||||
dependency_lines.extend(selected)
|
||||
recipe = ('[workspace]\nresolver = "2"\nmembers = ' +
|
||||
json.dumps(['crates/' + name for name in CRATES]) + '\n\n' + package +
|
||||
'\n\n[workspace.dependencies]\n' + '\n'.join(dependency_lines) + '\n').encode()
|
||||
return recipe
|
||||
|
||||
|
||||
def compare_lock(original, generated):
|
||||
original_blocks = original.decode().split('[[package]]')
|
||||
generated_blocks = generated.decode().split('[[package]]')
|
||||
require(original_blocks[0] == generated_blocks[0], 'same canonical lock version')
|
||||
known = {package_identity(lock_package(block)) for block in original_blocks[1:]}
|
||||
selected = [lock_package(block) for block in generated_blocks[1:]]
|
||||
require(len(selected) == len({package_identity(item) for item in selected}) and
|
||||
all(package_identity(item) in known for item in selected),
|
||||
'every generated package version/source/checksum exists in public original lock')
|
||||
require({item['name'] for item in selected if 'source' not in item} == set(PACKAGES),
|
||||
'only three original workspace packages')
|
||||
return len(selected)
|
||||
|
||||
|
||||
def generate_lock(case, payload, original):
|
||||
workspace = case / 'lock-workspace'
|
||||
workspace.mkdir(mode=0o700)
|
||||
for name, raw in payload.items():
|
||||
path = workspace / name
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
exclusive(path, raw)
|
||||
args = ['docker', 'run', '--rm', '--pull=never', '--network=none', '--read-only',
|
||||
'--tmpfs=/tmp:rw,nosuid,nodev,size=64m', '--cap-drop=ALL',
|
||||
'--security-opt=no-new-privileges', '--user', f'{os.getuid()}:{os.getgid()}',
|
||||
'-v', str(workspace) + ':/src:rw', '--entrypoint=cargo', LOCAL_BUILDER,
|
||||
'generate-lockfile', '--manifest-path', '/src/Cargo.toml', '--offline']
|
||||
result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, timeout=120, check=False)
|
||||
require(len(result.stdout) <= 1024 * 1024, 'bounded local Cargo lock generation output')
|
||||
exclusive(case / 'lock-generation.log', result.stdout)
|
||||
require(result.returncode == 0, 'local offline lock generation failed; retained original case log')
|
||||
os.chmod(workspace / 'Cargo.lock', 0o600)
|
||||
generated = regular(workspace / 'Cargo.lock')
|
||||
count = compare_lock(original, generated)
|
||||
# Cargo was allowed to alter only the generated lock file in this private recipe.
|
||||
for name, raw in payload.items():
|
||||
if name != 'Cargo.lock':
|
||||
require(regular(workspace / name) == raw, 'original copied source unchanged by lock generation')
|
||||
return generated, {'builder_image': LOCAL_BUILDER, 'offline': True,
|
||||
'package_count': count, 'log_sha256': digest(result.stdout)}
|
||||
|
||||
|
||||
def prepare(commit):
|
||||
require(re.fullmatch('[0-9a-f]{40}', commit) is not None, 'full public source commit')
|
||||
public = run(['git', '-C', str(ROOT), 'for-each-ref', '--format=%(objectname)', 'refs/remotes/origin/']).decode().splitlines()
|
||||
require(commit in public, 'checkpoint must be an already fetched origin branch tip')
|
||||
names = run(['git', '-C', str(ROOT), 'ls-tree', '-r', '--name-only', commit]).decode().splitlines()
|
||||
inputs = ['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml']
|
||||
inputs += [name for name in names if any(name == f'crates/{crate}/Cargo.toml' or
|
||||
(name.startswith(f'crates/{crate}/src/') and name.endswith('.rs')) for crate in CRATES)]
|
||||
require(len(inputs) >= 12 and len(inputs) <= 24, 'bounded complete three-crate source set')
|
||||
source = {}
|
||||
for name in sorted(inputs):
|
||||
raw = run(['git', '-C', str(ROOT), 'show', commit + ':' + name])
|
||||
require(regular(ROOT / name) == raw, 'working input differs from public checkpoint: ' + name)
|
||||
source[name] = raw
|
||||
payload = dict(source)
|
||||
payload['Cargo.toml'] = workspace_recipe(source['Cargo.toml'])
|
||||
payload['Cargo.lock'] = subset_lock(source['Cargo.lock'])
|
||||
payload['crates/time-source-linux/examples/operator_probe.rs'] = PROBE
|
||||
state = ROOT / 'devstate/server-time-source-bootstrap'
|
||||
state.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||
require(state.stat().st_uid == os.getuid() and stat.S_IMODE(state.stat().st_mode) == 0o700,
|
||||
'private local case root')
|
||||
case = state / str(uuid.uuid4())
|
||||
case.mkdir(mode=0o700)
|
||||
payload['Cargo.lock'], lock_proof = generate_lock(case, payload, source['Cargo.lock'])
|
||||
# Provenance originals are not compiled, but their exact bytes travel with the pack.
|
||||
for name in ('Cargo.toml', 'Cargo.lock'):
|
||||
payload['provenance/' + name] = source[name]
|
||||
manifest = {'profile': PROFILE, 'public_commit': commit, 'builder': BUILDER,
|
||||
'operator_sha256': digest(Path(__file__).read_bytes()),
|
||||
'lock_generation': lock_proof,
|
||||
'source_files': [{'path': name, 'bytes': len(raw), 'sha256': digest(raw)}
|
||||
for name, raw in sorted(source.items())],
|
||||
'package_files': [{'path': name, 'bytes': len(raw), 'sha256': digest(raw)}
|
||||
for name, raw in sorted(payload.items())],
|
||||
'probe_only': True, 'provisioned': False}
|
||||
payload['manifest.json'] = encoded(manifest)
|
||||
buffer = io.BytesIO()
|
||||
with tarfile.open(fileobj=buffer, mode='w', format=tarfile.USTAR_FORMAT) as archive:
|
||||
for name, raw in sorted(payload.items()):
|
||||
item = tarfile.TarInfo(name)
|
||||
item.size, item.mode, item.uid, item.gid, item.mtime = len(raw), 0o600, 0, 0, 0
|
||||
archive.addfile(item, io.BytesIO(raw))
|
||||
exclusive(case / 'source.tar', buffer.getvalue())
|
||||
exclusive(case / 'operator.py', Path(__file__).read_bytes())
|
||||
exclusive(case / 'manifest.json', encoded(manifest))
|
||||
receipt = {'case': case.name, 'source_tar_sha256': digest(buffer.getvalue()),
|
||||
'source_tar_bytes': len(buffer.getvalue()), 'manifest_sha256': digest(encoded(manifest)),
|
||||
'compiled': False, 'target_executed': False}
|
||||
exclusive(case / 'prepared.json', encoded(receipt))
|
||||
print(json.dumps({'path': str(case), **receipt}, sort_keys=True))
|
||||
|
||||
|
||||
def protected_dir(path, create=False):
|
||||
require(path.is_absolute(), 'absolute fixed operator path')
|
||||
current = Path('/')
|
||||
for part in path.parts[1:]:
|
||||
current /= part
|
||||
if create and not current.exists():
|
||||
current.mkdir(mode=0o700)
|
||||
info = current.lstat()
|
||||
require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and info.st_mode & 0o022 == 0,
|
||||
'root-owned protected directory ancestry')
|
||||
return path
|
||||
|
||||
|
||||
def host_case(value, create=False):
|
||||
require(os.geteuid() == 0 and os.uname().sysname == 'Linux' and os.uname().machine == 'aarch64',
|
||||
'root Linux ARM64 server only')
|
||||
protected_dir(HOST_ROOT, create=create)
|
||||
path = HOST_ROOT / identifier(value)
|
||||
if create:
|
||||
path.mkdir(mode=0o700)
|
||||
protected_dir(path)
|
||||
require(stat.S_IMODE(path.stat().st_mode) == 0o700, 'private operator case')
|
||||
return path
|
||||
|
||||
|
||||
def metadata(path):
|
||||
item = path.lstat()
|
||||
return {key: getattr(item, 'st_' + key) for key in ('dev', 'ino', 'uid', 'gid', 'mode', 'size', 'mtime_ns', 'ctime_ns')}
|
||||
|
||||
|
||||
def runtime_witness():
|
||||
names = ('gateway', 'api', 'postgres')
|
||||
result = {}
|
||||
for name in names:
|
||||
container = 'rust-discovery-server-' + name + '-1'
|
||||
# Never inspect Env, command arguments, health output, mounts or credentials.
|
||||
raw = run(['docker', 'inspect', '--format',
|
||||
'{{json .Image}} {{json .Config.User}} {{json .State.Running}} {{json .State.Pid}} {{json .State.StartedAt}} {{json .RestartCount}} {{json .NetworkSettings.Ports}}', container])
|
||||
# Multiple JSON values are deliberately decoded separately (spaces inside quoted dates/users are safe).
|
||||
decoder, values, text = json.JSONDecoder(), [], raw.decode().strip()
|
||||
while text:
|
||||
value, end = decoder.raw_decode(text)
|
||||
values.append(value)
|
||||
text = text[end:].lstrip()
|
||||
require(len(values) == 7 and values[2] is True and values[3] > 1, 'live expected runtime container')
|
||||
if name != 'postgres':
|
||||
require(values[0] == RUNTIME and values[1] == '10001:10001', 'fixed runtime image and user')
|
||||
for bindings in (values[6] or {}).values():
|
||||
require(bindings is None or (name != 'postgres' and all(x['HostIp'] == '127.0.0.1' for x in bindings)),
|
||||
'no public Docker port or PostgreSQL publication')
|
||||
result[name] = values
|
||||
return result
|
||||
|
||||
|
||||
def preflight():
|
||||
engine = run(['docker', 'version', '--format', '{{json .Server.Version}}']).decode().strip()
|
||||
image_values = json.loads(run(['docker', 'image', 'inspect', '--format',
|
||||
'[{{json .Id}},{{json .Architecture}},{{json .Os}}]', BUILDER]))
|
||||
image = dict(zip(('Id', 'Architecture', 'Os'), image_values, strict=True))
|
||||
# Inspect only public immutable image metadata, never Config/Env.
|
||||
require(image['Architecture'] == 'arm64' and image['Os'] == 'linux', 'cached ARM64 builder required')
|
||||
pid = int(run(['systemctl', 'show', 'chrony.service', '--property=MainPID', '--value']).strip())
|
||||
require(pid > 1, 'existing Chrony service PID')
|
||||
proc = Path('/proc') / str(pid)
|
||||
pidfd = os.pidfd_open(pid)
|
||||
try:
|
||||
status = regular(proc / 'status', 65536).decode()
|
||||
ids = {}
|
||||
for label in ('Uid', 'Gid'):
|
||||
match = re.search('^' + label + r':\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$', status, re.MULTILINE)
|
||||
require(match and len(set(match.groups())) == 1, 'fixed real/effective daemon IDs')
|
||||
ids[label.lower()] = int(match[1])
|
||||
birth = regular(proc / 'stat', 8192).decode().rsplit(')', 1)[1].split()[19]
|
||||
image_path = Path('/usr/sbin/chronyd')
|
||||
fixed = metadata(image_path)
|
||||
require(stat.S_ISREG(fixed['mode']) and fixed['uid'] == 0 and fixed['mode'] & 0o022 == 0,
|
||||
'protected Chrony executable')
|
||||
require(os.stat(proc / 'exe').st_ino == fixed['ino'] and os.stat(proc / 'exe').st_dev == fixed['dev'],
|
||||
'actual daemon fixed executable inode')
|
||||
image_hash = digest(regular(image_path))
|
||||
require(metadata(image_path) == fixed and
|
||||
regular(proc / 'stat', 8192).decode().rsplit(')', 1)[1].split()[19] == birth and
|
||||
int(run(['systemctl', 'show', 'chrony.service', '--property=MainPID', '--value']).strip()) == pid,
|
||||
'Chrony identity unchanged during probe')
|
||||
poll = __import__('select').poll()
|
||||
poll.register(pidfd, __import__('select').POLLIN)
|
||||
require(not poll.poll(0), 'Chrony process still alive')
|
||||
finally:
|
||||
os.close(pidfd)
|
||||
boot = regular(Path('/proc/sys/kernel/random/boot_id'), 128).decode().strip()
|
||||
require(str(uuid.UUID(boot)) == boot, 'kernel boot identity')
|
||||
return {'docker_engine_version': json.loads(engine),
|
||||
'builder': {key: image[key] for key in ('Id', 'Architecture', 'Os')},
|
||||
'runtime': runtime_witness(), 'boot_id': boot,
|
||||
'chrony_observation': {'schema_version': 1, 'daemon_pid': pid,
|
||||
'daemon_birth_ticks': int(birth), 'daemon_uid': ids['uid'], 'daemon_gid': ids['gid'],
|
||||
'daemon_image_sha256': image_hash},
|
||||
'chrony_image_metadata': fixed, 'pins_provisioned': False,
|
||||
'quality_sampled': False, 'registry_cache_verified': False}
|
||||
|
||||
|
||||
def unpack(case, expected):
|
||||
require(re.fullmatch('[0-9a-f]{64}', expected) is not None, 'exact operator-approved source hash')
|
||||
raw = regular(case / 'source.tar', 2 * 1024 * 1024)
|
||||
require(digest(raw) == expected, 'exact uploaded source archive')
|
||||
files = {}
|
||||
with tarfile.open(fileobj=io.BytesIO(raw), mode='r:') as archive:
|
||||
for member in archive.getmembers():
|
||||
require(member.isfile() and member.name not in files and member.size <= 1024 * 1024
|
||||
and not Path(member.name).is_absolute() and '..' not in Path(member.name).parts,
|
||||
'bounded regular unique archive entries')
|
||||
files[member.name] = archive.extractfile(member).read()
|
||||
require(len(files) <= 30 and 'manifest.json' in files, 'bounded source archive manifest')
|
||||
manifest = json.loads(files['manifest.json'])
|
||||
require(manifest['profile'] == PROFILE and manifest['builder'] == BUILDER and
|
||||
manifest['operator_sha256'] == digest(Path(__file__).read_bytes()), 'reviewed operator profile')
|
||||
require(set(files) == {'manifest.json'} | {x['path'] for x in manifest['package_files']}, 'exact manifest file set')
|
||||
originals = {x['path']: x for x in manifest['source_files']}
|
||||
for item in manifest['package_files']:
|
||||
require(len(files[item['path']]) == item['bytes'] and digest(files[item['path']]) == item['sha256'],
|
||||
'every source package hash and length')
|
||||
for name, item in originals.items():
|
||||
actual = files['provenance/' + name] if name in ('Cargo.toml', 'Cargo.lock') else files[name]
|
||||
require(len(actual) == item['bytes'] and digest(actual) == item['sha256'], 'unchanged original checkpoint input')
|
||||
require(files['Cargo.toml'] == workspace_recipe(files['provenance/Cargo.toml']) and
|
||||
compare_lock(files['provenance/Cargo.lock'], files['Cargo.lock']) == manifest['lock_generation']['package_count'] and
|
||||
manifest['lock_generation']['builder_image'] == LOCAL_BUILDER and
|
||||
manifest['lock_generation']['offline'] is True and
|
||||
files['crates/time-source-linux/examples/operator_probe.rs'] == PROBE,
|
||||
'exact generated recipe, lock closure, source-only probe')
|
||||
destination = case / 'workspace'
|
||||
destination.mkdir(mode=0o700)
|
||||
for name, value in sorted(files.items()):
|
||||
target = destination / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
exclusive(target, value)
|
||||
return {'source_tar_sha256': expected, 'manifest_sha256': digest(files['manifest.json']),
|
||||
'public_commit': manifest['public_commit'], 'package_files': manifest['package_files']}
|
||||
|
||||
|
||||
def verify_workspace(case):
|
||||
staged = json.loads(regular(case / 'stage' / 'result.json'))
|
||||
for item in staged['package_files']:
|
||||
raw = regular(case / 'workspace' / item['path'])
|
||||
require(digest(raw) == item['sha256'] and len(raw) == item['bytes'], 'original staged source bytes')
|
||||
return staged
|
||||
|
||||
|
||||
def long_command(args, log, timeout, cancel=None):
|
||||
fd = os.open(log, os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY, 0o600)
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
process = subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
start_new_session=True)
|
||||
deadline, written = time.monotonic() + timeout, 0
|
||||
try:
|
||||
with selectors.DefaultSelector() as selector:
|
||||
selector.register(process.stdout, selectors.EVENT_READ)
|
||||
while selector.get_map():
|
||||
remaining = deadline - time.monotonic()
|
||||
require(remaining > 0, 'bounded command timeout; read original case only')
|
||||
events = selector.select(remaining)
|
||||
require(events, 'bounded command timeout; read original case only')
|
||||
raw = os.read(process.stdout.fileno(), 65536)
|
||||
if not raw:
|
||||
selector.unregister(process.stdout)
|
||||
break
|
||||
allowed = 8 * 1024 * 1024 - written
|
||||
stream.write(raw[:allowed])
|
||||
written += min(len(raw), allowed)
|
||||
require(len(raw) <= allowed, 'bounded command output exceeded; read original case only')
|
||||
remaining = deadline - time.monotonic()
|
||||
require(remaining > 0, 'bounded command timeout; read original case only')
|
||||
code = process.wait(timeout=remaining)
|
||||
except BaseException:
|
||||
# The leader is still our unreaped child: its process group cannot
|
||||
# have been recycled to an unrelated process.
|
||||
if process.returncode is None:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
process.wait()
|
||||
if cancel is not None:
|
||||
cancel()
|
||||
raise
|
||||
finally:
|
||||
process.stdout.close()
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
return code
|
||||
|
||||
|
||||
def cancel_owned_builder(cidfile, image_id, case_id):
|
||||
# No name-based/global docker kill. Missing/uncertain identity is reported;
|
||||
# the independent in-container 850s timeout remains the final bound.
|
||||
if not cidfile.exists():
|
||||
raise RuntimeError('no retained container ID; read original case, internal timeout remains')
|
||||
cid = regular(cidfile, 128).decode().strip()
|
||||
require(re.fullmatch('[0-9a-f]{64}', cid) is not None, 'exact own Docker container ID')
|
||||
inspected = subprocess.run(['docker', 'inspect', '--format',
|
||||
'[{{json .Id}},{{json .Image}},{{json .Name}},{{json (index .Config.Labels "rmm.time-source.case")}},{{json .State.Running}}]', cid],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
if inspected.returncode != 0:
|
||||
# --rm may already have removed our completed container. No fallback.
|
||||
return
|
||||
require(len(inspected.stdout) <= 4096, 'bounded own container metadata')
|
||||
identity = json.loads(inspected.stdout)
|
||||
require(identity[:4] == [cid, image_id, '/rmm-time-source-build-' + case_id, case_id],
|
||||
'exact retained own container/image/case identity before cancellation')
|
||||
if identity[4] is True:
|
||||
run(['docker', 'kill', '--signal=KILL', cid], timeout=10, limit=4096)
|
||||
|
||||
|
||||
def elf(path):
|
||||
raw = regular(path, 32 * 1024 * 1024)
|
||||
require(raw[:6] == b'\x7fELF\x02\x01' and struct.unpack('<H', raw[18:20])[0] == 183,
|
||||
'actual ELF64 little-endian AArch64 artifact')
|
||||
return {'bytes': len(raw), 'sha256': digest(raw), 'elf_machine': 183}
|
||||
|
||||
|
||||
def build(case, phase, fetch):
|
||||
staged = verify_workspace(case)
|
||||
original = json.loads(regular(case / 'preflight/result.json'))
|
||||
before = preflight()
|
||||
require(before['builder'] == original['builder'] and before['boot_id'] == original['boot_id'] and
|
||||
original['builder']['Architecture'] == 'arm64' and original['builder']['Os'] == 'linux' and
|
||||
re.fullmatch('sha256:[0-9a-f]{64}', original['builder']['Id']),
|
||||
'original preflight immutable builder and boot; no silent tag reimage')
|
||||
image_id = original['builder']['Id']
|
||||
target = phase / 'target'
|
||||
target.mkdir(mode=0o700)
|
||||
cidfile = phase / 'container.id'
|
||||
# All Cargo lock/cache writes belong to this fresh bounded home. Offline
|
||||
# may read the cached image registry; explicit fetch gets an empty registry.
|
||||
setup = 'set -eu; '
|
||||
if not fetch:
|
||||
setup += 'if test -d /usr/local/cargo/registry; then ln -s /usr/local/cargo/registry /cargo/registry; fi; '
|
||||
setup += 'exec cargo "$@"'
|
||||
args = ['docker', 'run', '--rm', '--pull=never', '--network', 'bridge' if fetch else 'none',
|
||||
'--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--pids-limit=256',
|
||||
'--memory=2g', '--cpus=2', '--tmpfs=/tmp:rw,nosuid,nodev,size=256m',
|
||||
'--tmpfs=/cargo:rw,nosuid,nodev,size=512m,mode=0700', '-e', 'CARGO_HOME=/cargo',
|
||||
'-v', str(case / 'workspace') + ':/src:ro', '-v', str(target) + ':/out:rw',
|
||||
'--name=rmm-time-source-build-' + case.name,
|
||||
'--cidfile=' + str(cidfile),
|
||||
'--label=rmm.time-source.case=' + case.name,
|
||||
'-e', 'CARGO_TARGET_DIR=/out', '--entrypoint=/usr/bin/timeout', image_id,
|
||||
'--kill-after=3s', '850s', '/bin/sh', '-c', setup, 'rmm-fixed-cargo',
|
||||
'build', '--manifest-path', '/src/Cargo.toml', '--locked', '--release',
|
||||
'-p', 'rmm-time-broker-linux', '-p', 'rmm-time-source-linux', '--bins', '--examples']
|
||||
if not fetch:
|
||||
args.append('--offline')
|
||||
code = long_command(args, phase / 'build.log', 900,
|
||||
cancel=lambda: cancel_owned_builder(cidfile, image_id, case.name))
|
||||
after = preflight()
|
||||
require(before['runtime'] == after['runtime'] and before['boot_id'] == after['boot_id'],
|
||||
'RMM runtime metadata and boot preserved')
|
||||
verify_workspace(case)
|
||||
result = {'exit_code': code, 'registry_fetch_explicit': fetch, 'builder': before['builder'],
|
||||
'before': before, 'after': after, 'staged_manifest_sha256': staged['manifest_sha256'],
|
||||
'build_log_sha256': digest(regular(phase / 'build.log', 8 * 1024 * 1024)),
|
||||
'runtime_metadata_equal': True, 'native_source_quality_tested': False}
|
||||
if code == 0:
|
||||
# Cargo may hardlink release outputs to deps/. Copy exact immutable bytes
|
||||
# into fresh single-link root-owned artifacts before any probe execution.
|
||||
artifacts = phase / 'artifacts'
|
||||
artifacts.mkdir(mode=0o700)
|
||||
result['artifacts'] = {}
|
||||
for name in ('rmm-time-broker-linux', 'examples/operator_probe'):
|
||||
raw = regular(target / 'release' / name, 32 * 1024 * 1024, single_link=False)
|
||||
artifact = artifacts / Path(name).name
|
||||
exclusive(artifact, raw, mode=0o500)
|
||||
result['artifacts'][name] = elf(artifact)
|
||||
else:
|
||||
result['blocked'] = 'BUILD_FAILED_READ_LOG; offline cache is not assumed; no automatic retry/fetch'
|
||||
return result
|
||||
|
||||
|
||||
def probe(case, phase, expected):
|
||||
require(re.fullmatch('[0-9a-f]{64}', expected) is not None, 'operator-approved exact probe SHA')
|
||||
built = json.loads(regular(case / 'build' / 'result.json'))
|
||||
path = case / 'build/artifacts/operator_probe'
|
||||
artifact = elf(path)
|
||||
require(built['exit_code'] == 0 and artifact == built['artifacts']['examples/operator_probe']
|
||||
and artifact['sha256'] == expected, 'qualified original ARM64 source-only artifact')
|
||||
verify_workspace(case)
|
||||
# A previous provisioning action is a separate reviewed operation. Never synthesize pins here.
|
||||
protected_dir(Path('/etc/rmm-time-source'))
|
||||
pin = Path('/etc/rmm-time-source/chrony-pin.json')
|
||||
meta = metadata(pin)
|
||||
require(stat.S_ISREG(meta['mode']) and meta['uid'] == 0 and stat.S_IMODE(meta['mode']) == 0o600,
|
||||
'real protected Chrony pin required')
|
||||
protected_dir(Path('/run/rmm-time-source/clients'))
|
||||
before = preflight()
|
||||
args = ['systemd-run', '--quiet', '--wait', '--pipe', '--collect',
|
||||
'--unit=rmm-time-source-probe-' + case.name, '--property=Type=exec',
|
||||
'--property=User=root', '--property=Group=' + str(before['chrony_observation']['daemon_gid']),
|
||||
'--property=SupplementaryGroups=10001', '--property=NoNewPrivileges=yes',
|
||||
'--property=CapabilityBoundingSet=CAP_SYS_PTRACE', '--property=AmbientCapabilities=',
|
||||
'--property=ProtectSystem=strict', '--property=ProtectHome=yes',
|
||||
'--property=PrivateTmp=yes', '--property=PrivateDevices=yes',
|
||||
'--property=RestrictAddressFamilies=AF_UNIX', '--property=IPAddressDeny=any',
|
||||
'--property=SystemCallFilter=~@debug ptrace process_vm_readv process_vm_writev',
|
||||
'--property=RuntimeMaxSec=3s', '--property=ReadWritePaths=/run/rmm-time-source/clients', str(path)]
|
||||
code = long_command(args, phase / 'probe.log', 10)
|
||||
after = preflight()
|
||||
require(before['runtime'] == after['runtime'] and before['boot_id'] == after['boot_id'],
|
||||
'RMM runtime metadata and boot preserved')
|
||||
raw = regular(phase / 'probe.log', 16384)
|
||||
sample = None
|
||||
if code == 0:
|
||||
sample = json.loads(raw)
|
||||
require(sample['profile'] == 'rmm.native-source-quality.once.v1' and
|
||||
sample['clock_authority'] is False and sample['signed_attestation'] is False,
|
||||
'source-only single sample, no clock authority')
|
||||
return {'exit_code': code, 'artifact': artifact, 'before': before, 'after': after,
|
||||
'sample': sample, 'probe_log_sha256': digest(raw), 'runtime_metadata_equal': True,
|
||||
'signer_provisioned': False, 'broker_started': False, 'http_deployed': False}
|
||||
|
||||
|
||||
def host(args):
|
||||
case = host_case(args.case, create=args.command == 'host-preflight')
|
||||
if args.command == 'host-read':
|
||||
# No phase creation, Docker actions, signals or rerun. Read only original bounded case records.
|
||||
print(json.dumps({name: {'result': json.loads(regular(case / name / 'result.json'))
|
||||
if (case / name / 'result.json').exists() else None,
|
||||
'failure': json.loads(regular(case / name / 'failure.json'))
|
||||
if (case / name / 'failure.json').exists() else None,
|
||||
'intent_present': (case / name / 'intent.json').exists()}
|
||||
for name in ('preflight', 'stage', 'build', 'probe')}, sort_keys=True))
|
||||
return
|
||||
name = args.command.removeprefix('host-')
|
||||
phase = case / name
|
||||
phase.mkdir(mode=0o700) # An existing intent OR result never permits rerun.
|
||||
exclusive(phase / 'intent.json', encoded({'profile': PROFILE, 'case': case.name, 'phase': name,
|
||||
'operator_sha256': digest(Path(__file__).read_bytes()), 'started_unix_ns': time.time_ns()}))
|
||||
try:
|
||||
if name == 'preflight':
|
||||
result = preflight()
|
||||
elif name == 'stage':
|
||||
result = unpack(case, args.sha256)
|
||||
elif name == 'build':
|
||||
result = build(case, phase, args.registry_fetch)
|
||||
else:
|
||||
result = probe(case, phase, args.sha256)
|
||||
exclusive(phase / 'result.json', encoded(result))
|
||||
print(json.dumps({'case': case.name, 'phase': name, 'result': result}, sort_keys=True))
|
||||
except Exception as error:
|
||||
# Fixed controlled messages only; never write exception repr, paths from secrets, nested errors or env.
|
||||
exclusive(phase / 'failure.json', encoded({'failed': True, 'exception_type': type(error).__name__,
|
||||
'read_original_case_only': True}))
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest='command', required=True)
|
||||
prepare_parser = commands.add_parser('prepare')
|
||||
prepare_parser.add_argument('--commit', required=True)
|
||||
for name in ('preflight', 'stage', 'build', 'probe', 'read'):
|
||||
command = commands.add_parser('host-' + name)
|
||||
command.add_argument('--case', required=True)
|
||||
if name in ('stage', 'probe'):
|
||||
command.add_argument('--sha256', required=True)
|
||||
if name == 'build':
|
||||
command.add_argument('--registry-fetch', action='store_true',
|
||||
help='explicit operator-authorized crates.io fetch using locked checksums; never OS updates')
|
||||
args = parser.parse_args()
|
||||
if args.command == 'prepare':
|
||||
prepare(args.commit)
|
||||
else:
|
||||
host(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user