486 lines
38 KiB
Python
486 lines
38 KiB
Python
#!/usr/bin/env python3
|
|
"""Package24 pure profile contracts: Linux VM FIRST, then Windows VM.
|
|
|
|
Only two fixed test executables run in separate bootstrap scratch directories.
|
|
No RMM job, daemon, live database, PKI, policy, enrollment or capability change.
|
|
Native execution is explicit; any lost reply permits same-case readback only.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import datetime as dt
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import uuid
|
|
|
|
import vm_access_check as access
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
STATE = ROOT / 'devstate/windows-profile'
|
|
IMAGE = 'rust-discovery-durable-final-build:local'
|
|
PROFILE = 'windows-process-profile-v1'
|
|
LINUX_REMOTE = '/var/lib/rmm-windows-profile-units'
|
|
KINDS = ('domain', 'contracts')
|
|
COUNTS = {'linux': {'domain': 16, 'contracts': 7}, 'windows': {'domain': 16, 'contracts': 6}}
|
|
DOMAIN_NAMES = sorted(('apt_transaction::tests::closure_bytes_archives_and_code_are_bound',
|
|
'apt_transaction::tests::helper_schema_two_roundtrip_and_strict_action_binding',
|
|
'inventory_document::tests::additive_metadata_and_bounds',
|
|
'inventory_document::tests::exact_linux_recovery_scope_does_not_admit_other_hyphens_or_windows',
|
|
'inventory_document::tests::windows_scope_does_not_weaken_linux_or_accept_platform_spoofing',
|
|
'jobs::tests::additive_recipe_refs_preserve_legacy_plan_bytes', 'jobs::tests::codec_golden_and_exact_script_bytes',
|
|
'jobs::tests::no_terminal_replay_or_unknown_restart', 'jobs::tests::permit_codec_golden_all_bindings',
|
|
'jobs::tests::updates_require_exact_versions_and_never_implicit_reboot',
|
|
'jobs::tests::windows_interpreter_and_capability_contract',
|
|
'jobs::tests::windows_remote_signed_profile_is_optional_exact_and_capability_bound',
|
|
'jobs::tests::windows_system_scm_is_explicit_and_preserves_linux_service_bytes',
|
|
'reconciliation::tests::outcome_codec_keeps_raw_output_and_sorts_evidence',
|
|
'reconciliation::tests::reconciliation_cannot_be_a_permit_or_unbounded_ticket',
|
|
'tests::production_never_accepts_development_auth'))
|
|
CONTRACT_NAMES = sorted(('atomic_begin_execution_rolls_back_both_rows_on_missing_witness_or_start_insert_failure',
|
|
'completion_cas_preserves_original_bytes_and_recovers_only_verified_bound_evidence',
|
|
'consumed_attempt_and_unknown_lock_survive_reopen_without_reexecution',
|
|
'exact_permit_bytes_time_and_all_identity_hash_bindings_are_required',
|
|
'historical_request_and_binding_codecs_keep_field_order_arrays_and_strictness',
|
|
'openssl_and_portable_ed25519_agree_on_current_keys_bytes_and_tampering',
|
|
'portable_acceptance_requires_platform_capability_and_exact_redelivery'))
|
|
UNIX_ONLY = 'openssl_and_portable_ed25519_agree_on_current_keys_bytes_and_tampering'
|
|
EXPECTED = {'linux': {'domain': DOMAIN_NAMES, 'contracts': CONTRACT_NAMES},
|
|
'windows': {'domain': DOMAIN_NAMES, 'contracts': [x for x in CONTRACT_NAMES if x != UNIX_ONLY]}}
|
|
|
|
|
|
def require(value, code):
|
|
if not value:
|
|
raise RuntimeError(code)
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def object_pairs(items):
|
|
value = {}
|
|
for key, item in items:
|
|
require(key not in value, 'duplicate JSON field')
|
|
value[key] = item
|
|
return value
|
|
|
|
|
|
def private(path, directory=False):
|
|
item = Path(path).lstat()
|
|
require(not stat.S_ISLNK(item.st_mode) and item.st_uid == os.getuid()
|
|
and stat.S_IMODE(item.st_mode) == (0o700 if directory else 0o600)
|
|
and (stat.S_ISDIR(item.st_mode) if directory else stat.S_ISREG(item.st_mode) and item.st_nlink == 1),
|
|
'private owned regular path')
|
|
|
|
|
|
def load(path):
|
|
private(path)
|
|
require(Path(path).stat().st_size <= 2 * 1024 * 1024, 'bounded private JSON')
|
|
return json.loads(Path(path).read_bytes(), object_pairs_hook=object_pairs)
|
|
|
|
|
|
def save(path, value):
|
|
private(Path(path).parent, True)
|
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY, 0o600)
|
|
with os.fdopen(fd, 'w') as stream:
|
|
json.dump(value, stream, indent=2); stream.write('\n'); stream.flush(); os.fsync(stream.fileno())
|
|
|
|
|
|
def setup_state():
|
|
os.umask(0o077)
|
|
STATE.parent.mkdir(mode=0o700, exist_ok=True)
|
|
metadata = STATE.parent.lstat()
|
|
require(stat.S_ISDIR(metadata.st_mode) and metadata.st_uid == os.getuid() and not metadata.st_mode & 0o002,
|
|
'owned non-world-writable devstate parent')
|
|
STATE.mkdir(mode=0o700, exist_ok=True)
|
|
private(STATE, True)
|
|
|
|
|
|
def source_files():
|
|
paths = [ROOT / p for p in ('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml',
|
|
'deploy/windows_profile_acceptance.py', 'deploy/remote_client_bootstrap.py',
|
|
'deploy/vm_access_check.py', 'deploy/vm_mvp_acceptance.py', 'deploy/windows/client_bootstrap.py')]
|
|
paths += list((ROOT / 'crates').rglob('Cargo.toml'))
|
|
for name in ('domain', 'agent-core', 'protocol'):
|
|
paths += [p for p in (ROOT / 'crates' / name).rglob('*') if p.is_file()]
|
|
paths += [p for p in (ROOT / 'proto').rglob('*') if p.is_file()]
|
|
for path in paths:
|
|
require(not path.is_symlink() and path.is_file(), 'regular source inputs only')
|
|
return {str(p.relative_to(ROOT)): sha(p) for p in sorted(set(paths))}
|
|
|
|
|
|
def names(raw):
|
|
return sorted(re.findall(r'^([A-Za-z0-9_:]+): test\r?$', raw, re.M))
|
|
|
|
|
|
def check_list(raw, expected):
|
|
require(names(raw) == expected and len(set(expected)) == len(expected), 'all exact test names before execution')
|
|
require(all(not line.strip() or re.fullmatch(r'[A-Za-z0-9_:]+: test', line) for line in raw.splitlines()),
|
|
'terse list contains only exact tests, no benchmarks or extra entries')
|
|
|
|
|
|
def check_output(raw, count):
|
|
require(re.search(rb'^test result: ok\. ' + str(count).encode()
|
|
+ rb' passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;[^\r\n]*\r?$', raw, re.M),
|
|
'all tests passed without skipped or filtered tests')
|
|
|
|
|
|
def command(args, folder, timeout=300):
|
|
index = len(list(folder.glob('command-*.json')))
|
|
try:
|
|
done = subprocess.run(args, capture_output=True, timeout=timeout)
|
|
save(folder / ('command-%03d.json' % index), dict(returncode=done.returncode,
|
|
stdout_b64=base64.b64encode(done.stdout).decode(), stderr_b64=base64.b64encode(done.stderr).decode()))
|
|
require(done.returncode == 0, 'local qualification command failed; private output retained')
|
|
return done.stdout
|
|
except subprocess.TimeoutExpired as error:
|
|
save(folder / ('command-%03d.json' % index), dict(timed_out=True,
|
|
stdout_b64=base64.b64encode(error.stdout or b'').decode(), stderr_b64=base64.b64encode(error.stderr or b'').decode()))
|
|
raise RuntimeError('local qualification timed out; no automatic retry') from None
|
|
|
|
|
|
def scripts():
|
|
return {'linux_case_sha256': hashlib.sha256(inspect.getsource(linux_case).encode()).hexdigest(),
|
|
'windows_script_sha256': hashlib.sha256(WINDOWS.encode()).hexdigest()}
|
|
|
|
|
|
def build():
|
|
setup_state()
|
|
folder = STATE / ('build-' + str(uuid.uuid4())); folder.mkdir(mode=0o700)
|
|
before = source_files(); save(folder / 'source-before.json', before)
|
|
image = command(['docker', 'image', 'inspect', '--format', '{{.Id}}', IMAGE], folder).decode().strip()
|
|
require(re.fullmatch(r'sha256:[a-f0-9]{64}', image), 'immutable local build image')
|
|
target = STATE / 'target'; target.mkdir(mode=0o700, exist_ok=True); private(target, True)
|
|
common = ['docker', 'run', '--rm', '--network', 'none', '--read-only', '--tmpfs', '/tmp:rw,size=512m',
|
|
'-v', str(ROOT) + ':/src:ro', '-v', str(target) + ':/target', '-e', 'CARGO_TARGET_DIR=/target', '--entrypoint', 'cargo', image]
|
|
artifacts = {'linux': {}, 'windows': {}}
|
|
for platform, triple in [('linux', 'x86_64-unknown-linux-gnu'), ('windows', 'x86_64-pc-windows-gnu')]:
|
|
for kind in KINDS:
|
|
package, selection = ('rmm-domain', ['--lib']) if kind == 'domain' else ('rmm-agent-core', ['--test', 'contracts'])
|
|
options = ['--manifest-path', '/src/Cargo.toml', '--locked', '--offline', '--target', triple, '-p', package]
|
|
command(common + ['clippy'] + options + selection + ['--', '-D', 'warnings'], folder)
|
|
raw = command(common + ['test'] + options + selection + ['--release', '--no-run', '--message-format=json'], folder)
|
|
linked = []
|
|
for line in raw.splitlines():
|
|
try: item = json.loads(line)
|
|
except ValueError: continue
|
|
expected_name = 'rmm_domain' if kind == 'domain' else 'contracts'
|
|
if item.get('reason') == 'compiler-artifact' and item.get('executable') and item.get('profile', {}).get('test') and item['target']['name'] == expected_name:
|
|
linked.append(target / Path(item['executable']).relative_to('/target'))
|
|
require(len(linked) == 1, 'exact selected unit executable')
|
|
destination = folder / platform; destination.mkdir(mode=0o700, exist_ok=True)
|
|
original = destination / (kind + '.linked')
|
|
delivery = destination / (kind + ('.exe' if platform == 'windows' else ''))
|
|
for path in (original, delivery): shutil.copyfile(linked[0], path); path.chmod(0o600)
|
|
linked_sha = sha(original)
|
|
tool = 'x86_64-w64-mingw32-strip' if platform == 'windows' else 'strip'
|
|
command(['docker', 'run', '--rm', '--network', 'none', '--read-only', '-v', str(destination) + ':/artifacts',
|
|
'--entrypoint', tool, image, '--strip-debug', '/artifacts/' + delivery.name], folder)
|
|
require(sha(linked[0]) == sha(original) == linked_sha, 'unstripped original link preserved')
|
|
data = delivery.read_bytes()
|
|
if platform == 'linux':
|
|
require(data[:6] == b'\x7fELF\x02\x01' and data[18:20] == b'\x3e\0', 'Linux x64 ELF')
|
|
else:
|
|
offset = int.from_bytes(data[60:64], 'little')
|
|
require(data[:2] == b'MZ' and data[offset:offset + 6] == b'PE\0\0\x64\x86', 'Windows x64 PE')
|
|
artifacts[platform][kind] = dict(path=str(delivery.resolve()), sha256=sha(delivery), bytes=len(data),
|
|
original_link_path=str(original.resolve()), original_link_sha256=linked_sha, debug_stripped_delivery=True)
|
|
# Run exactly the deliverable Linux bytes in an offline disposable container.
|
|
for kind, item in artifacts['linux'].items():
|
|
invoke = ['docker', 'run', '--rm', '--network', 'none', '--read-only', '--tmpfs', '/tmp:rw,exec,size=128m',
|
|
'-v', item['path'] + ':/unit:ro', '--entrypoint', 'python3', image, '-c']
|
|
for mode in ('list', 'run'):
|
|
args = ['--list', '--format', 'terse'] if mode == 'list' else ['--test-threads=1']
|
|
script = 'import shutil,os,subprocess;shutil.copyfile("/unit","/tmp/unit");os.chmod("/tmp/unit",0o500);subprocess.run(["/tmp/unit"]+' + repr(args) + ',check=True,cwd="/tmp",env={"PATH":"/usr/bin:/bin","LANG":"C.UTF-8","TMPDIR":"/tmp","RUST_BACKTRACE":"0"})'
|
|
raw = command(invoke + [script], folder)
|
|
save(folder / ('local-linux-' + kind + '-' + mode + '.json'), dict(stdout_b64=base64.b64encode(raw).decode()))
|
|
if mode == 'list': check_list(raw.decode(), EXPECTED['linux'][kind])
|
|
else: check_output(raw, COUNTS['linux'][kind])
|
|
dynamic = command(['docker', 'run', '--rm', '--network', 'none', '--read-only', '-v', item['path'] + ':/unit:ro',
|
|
'--entrypoint', 'readelf', image, '-d', '/unit'], folder).decode()
|
|
versions = command(['docker', 'run', '--rm', '--network', 'none', '--read-only', '-v', item['path'] + ':/unit:ro',
|
|
'--entrypoint', 'readelf', image, '--version-info', '/unit'], folder).decode()
|
|
glibc = sorted(set(re.findall(r'GLIBC_(\d+\.\d+)', versions)), key=lambda x: tuple(map(int, x.split('.'))))
|
|
require(glibc and tuple(map(int, glibc[-1].split('.'))) <= (2, 39), 'VM-compatible glibc requirement')
|
|
item['glibc_max'] = glibc[-1]; item['needed'] = sorted(re.findall(r'Shared library: \[([^\]]+)\]', dynamic))
|
|
require(source_files() == before, 'source and runner unchanged through qualification')
|
|
manifest = dict(schema_version=1, profile=PROFILE, source_files=before, source_before_after_equal=True,
|
|
runner_sha256=sha(__file__), scripts=scripts(), test_names=EXPECTED, test_counts=COUNTS, artifacts=artifacts,
|
|
build_image_id=image, both_targets_clippy_D_warnings=True, linux_lists_actual=True,
|
|
windows_lists_expected_until_native=True, local_linux_tests_passed=23, native_linux_executed=False,
|
|
native_windows_executed=False, product_state_writes=False, rmm_jobs=False)
|
|
save(folder / 'manifest.json', manifest)
|
|
print(json.dumps(dict(manifest=str((folder / 'manifest.json').resolve()), manifest_sha256=sha(folder / 'manifest.json'),
|
|
linux_tests=23, windows_tests_expected=22, native_executed=False)))
|
|
|
|
|
|
def verified(path, new_run):
|
|
path = path.absolute(); private(path.parent, True)
|
|
require(path.name == 'manifest.json' and path.parent.parent == STATE and re.fullmatch(r'build-[0-9a-f-]{36}', path.parent.name), 'fixed build manifest path')
|
|
value = load(path)
|
|
require(value['schema_version'] == 1 and value['profile'] == PROFILE and value['runner_sha256'] == sha(__file__)
|
|
and value['scripts'] == scripts() and value['test_names'] == EXPECTED and value['test_counts'] == COUNTS
|
|
and value['source_before_after_equal'] is True and value['both_targets_clippy_D_warnings'] is True,
|
|
'original manifest/runner/remote scripts/profile binding')
|
|
if new_run: require(value['source_files'] == source_files(), 'new execution requires same current qualified source')
|
|
for platform in COUNTS:
|
|
require(set(value['artifacts'][platform]) == set(KINDS), 'exact two artifacts')
|
|
for kind, item in value['artifacts'][platform].items():
|
|
suffix = '.exe' if platform == 'windows' else ''
|
|
require(Path(item['path']) == path.parent / platform / (kind + suffix)
|
|
and Path(item['original_link_path']) == path.parent / platform / (kind + '.linked'), 'fixed private artifact paths')
|
|
for field, digest in [('path', item['sha256']), ('original_link_path', item['original_link_sha256'])]:
|
|
private(item[field]); require(sha(item[field]) == digest, 'immutable original/delivery bytes')
|
|
require(Path(item['path']).stat().st_size == item['bytes'], 'artifact size unchanged')
|
|
return value
|
|
|
|
|
|
def access_settings(root, platform):
|
|
"""Read strictly parsed existing access only when the parent executes native."""
|
|
require(root is not None and not root.is_symlink(), 'explicit real access-root required')
|
|
root = root.absolute(); require(root.is_dir() and root.resolve() == root, 'canonical access-root')
|
|
if platform == 'linux':
|
|
cfg = access.load_config(root / 'credentials/linux-vm.conf')
|
|
require(cfg['address'] == '116.203.20.85' and cfg['username'] == 'root' and cfg.get('port', '22') == '22'
|
|
and cfg.get('ssh_key_path'), 'fixed Cloud Linux bootstrap target')
|
|
prefix = ['ssh', '-F', '/dev/null', '-T', '-p', '22', '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes',
|
|
'-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10', '-i', str(Path(cfg['ssh_key_path']).expanduser()), 'root@116.203.20.85']
|
|
else:
|
|
cfg = access.load_config(root / 'credentials/server.conf')
|
|
require(cfg['address'] == '88.99.80.202' and cfg['username'] == 'root' and cfg.get('port', '22') == '22'
|
|
and cfg.get('ssh_key_path'), 'fixed server reverse-jump only')
|
|
credentials = root / 'credentials'; private(credentials, True)
|
|
record = load(credentials / 'windows-dev-access.json')
|
|
require(set(record) == {'ssh_prefix', 'expires_at'}, 'known Windows access schema')
|
|
expires = dt.datetime.fromisoformat(record['expires_at'])
|
|
require(expires.utcoffset() is not None and dt.datetime.now(dt.timezone.utc) < expires, 'existing dev access still valid')
|
|
key = str(Path(cfg['ssh_key_path']).expanduser())
|
|
jump = ['ssh', '-F', '/dev/null', '-T', '-p', '22', '-i', key, '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes',
|
|
'-o', 'ForwardAgent=no', '-o', 'IdentitiesOnly=yes', '-W', '127.0.0.1:22240', 'root@88.99.80.202']
|
|
prefix = ['ssh', '-F', '/dev/null', '-T', '-i', key, '-o', 'ProxyCommand=' + shlex.join(jump),
|
|
'-o', 'HostKeyAlias=rmm-windows-dev', '-o', 'UserKnownHostsFile=' + str(credentials / 'windows-dev.known_hosts'),
|
|
'-o', 'GlobalKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes',
|
|
'-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10', 'rmm-dev@127.0.0.1']
|
|
require(record['ssh_prefix'] == prefix, 'exact registered pinned reverse SSH prefix')
|
|
private(credentials / 'windows-dev.known_hosts')
|
|
return prefix
|
|
|
|
|
|
def transport(prefix, command_text, data, folder, timeout=60):
|
|
"""No stdout/stderr escapes the private case directory on transport failure."""
|
|
try:
|
|
result = subprocess.run(prefix + [command_text], input=data, capture_output=True, timeout=timeout)
|
|
save(folder / ('transport-' + str(uuid.uuid4()) + '.json'), dict(returncode=result.returncode,
|
|
stdout_b64=base64.b64encode(result.stdout).decode(), stderr_b64=base64.b64encode(result.stderr).decode()))
|
|
require(result.returncode == 0, 'native transport incomplete; same-case readback only')
|
|
value = json.loads(result.stdout.decode('utf-8-sig'), object_pairs_hook=object_pairs)
|
|
require(isinstance(value, dict) and 'bootstrap_error' not in value, 'native bootstrap incomplete; private receipt retained')
|
|
return value
|
|
except subprocess.TimeoutExpired as error:
|
|
save(folder / ('transport-timeout-' + str(uuid.uuid4()) + '.json'), dict(timed_out=True,
|
|
stdout_b64=base64.b64encode(error.stdout or b'').decode(), stderr_b64=base64.b64encode(error.stderr or b'').decode()))
|
|
raise RuntimeError('native reply lost; read original case only') from None
|
|
|
|
|
|
def linux_rpc(prefix, payload, folder):
|
|
code = 'import os,json,subprocess,base64,hashlib,uuid\nfrom pathlib import Path\n' + inspect.getsource(require) + '\n' + inspect.getsource(linux_case)
|
|
code += '\ntry:\n print(json.dumps(linux_case(' + repr(payload) + ')))\nexcept Exception as e:\n print(json.dumps(dict(bootstrap_error=True,error_class=type(e).__name__)))\n'
|
|
return transport(prefix, 'python3 -', code.encode(), folder)
|
|
|
|
|
|
def windows_rpc(prefix, payload, folder):
|
|
script = 'try {\n' + WINDOWS + '\n} catch {[pscustomobject]@{bootstrap_error=$true;error_id=$_.FullyQualifiedErrorId;line=$_.InvocationInfo.ScriptLineNumber}|ConvertTo-Json -Compress;exit 1}'
|
|
command_text = 'powershell.exe -NoLogo -NoProfile -NonInteractive -EncodedCommand ' + base64.b64encode(script.encode('utf-16le')).decode()
|
|
return transport(prefix, command_text, (json.dumps(payload) + '\n').encode(), folder)
|
|
|
|
|
|
def upload(prefix, item, destination, folder, kind):
|
|
args = ['sftp'] + [part for part in prefix[1:-1] if part != '-T'] + ['-b', '-', prefix[-1]]
|
|
# Linux sftp uses uppercase -P; Windows prefix has no -p option.
|
|
if '-p' in args: args[args.index('-p')] = '-P'
|
|
batch = ('put ' + json.dumps(item['path']) + ' ' + json.dumps(destination) + '\n').encode()
|
|
save(folder / ('transfer-' + kind + '-intent.json'), dict(sha256=item['sha256'], bytes=item['bytes']))
|
|
try:
|
|
result = subprocess.run(args, input=batch, capture_output=True, timeout=60)
|
|
save(folder / ('transfer-' + kind + '-result.json'), dict(returncode=result.returncode,
|
|
stdout_b64=base64.b64encode(result.stdout).decode(), stderr_b64=base64.b64encode(result.stderr).decode()))
|
|
require(result.returncode == 0, 'fixed artifact transfer failed; no retry')
|
|
except subprocess.TimeoutExpired as error:
|
|
save(folder / ('transfer-' + kind + '-timeout.json'), dict(timed_out=True,
|
|
stderr_b64=base64.b64encode(error.stderr or b'').decode()))
|
|
raise RuntimeError('artifact transfer incomplete; no retry') from None
|
|
|
|
|
|
def linux_case(p):
|
|
import resource
|
|
import stat
|
|
ident = str(uuid.UUID(p['id']))
|
|
require(ident == p['id'] and p['action'] in ('prepare', 'run', 'read') and p['profile'] == 'windows-process-profile-v1', 'fixed unit case')
|
|
require(os.geteuid() == 0 and os.uname().machine == 'x86_64' and set(p['artifacts']) == {'domain', 'contracts'}, 'fixed Linux root qualification')
|
|
root = Path('/var/lib/rmm-windows-profile-units')
|
|
def safe(path, directory):
|
|
s = path.lstat()
|
|
require(s.st_uid == 0 and stat.S_IMODE(s.st_mode) == (0o700 if directory else 0o600)
|
|
and (stat.S_ISDIR(s.st_mode) if directory else stat.S_ISREG(s.st_mode) and s.st_nlink == 1), 'private fixed unit path')
|
|
def write(path, value):
|
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'w') as out: json.dump(value, out); out.flush(); os.fsync(out.fileno())
|
|
os.umask(0o077)
|
|
case = root / ident
|
|
if p['action'] == 'read' and not case.exists(): return dict(id=ident, status='remote_case_absent')
|
|
if p['action'] == 'prepare':
|
|
if not root.exists(): root.mkdir(mode=0o700)
|
|
safe(root, True); case.mkdir(mode=0o700); write(case / 'intent.json', p)
|
|
return dict(id=ident, prepared=True)
|
|
safe(root, True); safe(case, True); safe(case / 'intent.json', False)
|
|
original = json.loads((case / 'intent.json').read_bytes())
|
|
require({k: v for k, v in p.items() if k != 'action'} == {k: v for k, v in original.items() if k != 'action'}, 'original complete unit intent')
|
|
if p['action'] == 'read':
|
|
if not (case / 'result.json').exists(): return dict(id=ident, status='started_without_result' if (case / 'started.json').exists() else 'prepared_without_start')
|
|
safe(case / 'result.json', False); return json.loads((case / 'result.json').read_bytes())
|
|
write(case / 'started.json', dict(manifest_sha256=p['manifest_sha256'], execute_once=True))
|
|
temp = case / 'tmp'; temp.mkdir(mode=0o700)
|
|
env = {'PATH': '/usr/bin:/bin', 'LANG': 'C.UTF-8', 'LC_ALL': 'C.UTF-8', 'TMPDIR': str(temp), 'RUST_BACKTRACE': '0'}
|
|
def limits():
|
|
resource.setrlimit(resource.RLIMIT_CORE, (0, 0)); resource.setrlimit(resource.RLIMIT_CPU, (15, 15)); resource.setrlimit(resource.RLIMIT_FSIZE, (1048576, 1048576))
|
|
require(subprocess.run(['/usr/bin/unshare', '--net', '--', '/usr/bin/true'], env=env, capture_output=True, timeout=3).returncode == 0, 'network namespace required')
|
|
listings = {}
|
|
for kind in ('domain', 'contracts'):
|
|
binary = case / kind; safe(binary, False)
|
|
require(hashlib.sha256(binary.read_bytes()).hexdigest() == p['artifacts'][kind]['sha256'], 'exact Linux delivery')
|
|
binary.chmod(0o500)
|
|
listed = subprocess.run(['/usr/bin/unshare', '--net', '--', str(binary), '--list', '--format', 'terse'], cwd=case, env=env, capture_output=True, timeout=5, preexec_fn=limits)
|
|
require(listed.returncode == 0 and len(listed.stdout) <= 65536 and len(listed.stderr) <= 65536, 'bounded actual listing')
|
|
actual = sorted(line[:-6] for line in listed.stdout.decode().splitlines() if line.endswith(': test'))
|
|
require(all(not line.strip() or __import__('re').fullmatch(r'[A-Za-z0-9_:]+: test', line) for line in listed.stdout.decode().splitlines())
|
|
and actual == p['names'][kind] and len(actual) == {'domain': 16, 'contracts': 7}[kind], 'exact native test names before ANY test execution')
|
|
listings[kind] = actual
|
|
reports = {}
|
|
for kind in ('domain', 'contracts'):
|
|
with (case / (kind + '-stdout.bin')).open('xb') as out, (case / (kind + '-stderr.bin')).open('xb') as err:
|
|
timed = False
|
|
try: code = subprocess.run(['/usr/bin/unshare', '--net', '--', str(case / kind), '--test-threads=1'], cwd=case, env=env,
|
|
stdin=subprocess.DEVNULL, stdout=out, stderr=err, timeout=18, preexec_fn=limits).returncode
|
|
except subprocess.TimeoutExpired: timed = True; code = None
|
|
out.flush(); err.flush(); os.fsync(out.fileno()); os.fsync(err.fileno())
|
|
raw, errors = [(case / (kind + '-' + stream + '.bin')).read_bytes() for stream in ('stdout', 'stderr')]
|
|
require(len(raw) <= 1048576 and len(errors) <= 1048576, 'bounded native output')
|
|
reports[kind] = dict(sha256=p['artifacts'][kind]['sha256'], names=listings[kind], exit_code=code, timed_out=timed,
|
|
stdout_b64=base64.b64encode(raw).decode(), stderr_b64=base64.b64encode(errors).decode())
|
|
write(case / (kind + '-result.json'), reports[kind])
|
|
value = dict(id=ident, platform='linux', profile=p['profile'], manifest_sha256=p['manifest_sha256'], runner_sha256=p['runner_sha256'],
|
|
scripts=p['scripts'], reports=reports, network_namespace=True, environment_cleared=True, product_state_writes=False, rmm_jobs=False)
|
|
write(case / 'result.json', value); return value
|
|
|
|
|
|
WINDOWS = r'''
|
|
$ErrorActionPreference='Stop';$p=([Console]::In.ReadLine()|ConvertFrom-Json)
|
|
if([string]([guid]$p.id)-cne$p.id-or$p.action-notin@('prepare','run','read')-or$p.profile-cne'windows-process-profile-v1'-or$p.manifest_sha256-notmatch'^[a-f0-9]{64}$'-or$p.linux_proof_sha256-notmatch'^[a-f0-9]{64}$'){throw 'Fixed Linux-first profile case'}
|
|
$root='C:\ProgramData\RustDiscoveryBootstrap';$case=Join-Path $root ('windows-profile-'+$p.id)
|
|
function Safe($path,$dir){$i=Get-Item -LiteralPath $path -Force;if($i.PSIsContainer-ne$dir-or($i.Attributes-band[IO.FileAttributes]::ReparsePoint)){throw 'Unit path type'};$a=Get-Acl -LiteralPath $path;if($a.GetOwner([Security.Principal.SecurityIdentifier]).Value-notin@('S-1-5-18','S-1-5-32-544')){throw 'Private unit owner'};$rules=@($a.GetAccessRules($true,$true,[Security.Principal.SecurityIdentifier]));if(!$rules.Count){throw 'Empty private ACL'};foreach($e in $rules){if($e.AccessControlType-ne'Allow'-or$e.IdentityReference.Value-notin@('S-1-5-18','S-1-5-32-544')-or($e.PropagationFlags-band[Security.AccessControl.PropagationFlags]::InheritOnly)){throw 'Broad private unit ACL'}}}
|
|
function WriteNew($path,$value){$f=[IO.File]::Open($path,[IO.FileMode]::CreateNew,[IO.FileAccess]::Write,[IO.FileShare]::Read);try{$b=[Text.Encoding]::UTF8.GetBytes(($value|ConvertTo-Json -Depth 16 -Compress));$f.Write($b,0,$b.Length);$f.Flush($true)}finally{$f.Dispose()}}
|
|
function MatchIntent($old){if($old.id-cne$p.id-or$old.profile-cne$p.profile-or$old.manifest_sha256-cne$p.manifest_sha256-or$old.runner_sha256-cne$p.runner_sha256-or$old.linux_proof_sha256-cne$p.linux_proof_sha256-or$old.scripts.linux_case_sha256-cne$p.scripts.linux_case_sha256-or$old.scripts.windows_script_sha256-cne$p.scripts.windows_script_sha256){throw 'Original profile intent'};foreach($kind in @('domain','contracts')){if($old.artifacts.$kind.sha256-cne$p.artifacts.$kind.sha256-or$old.artifacts.$kind.bytes-ne$p.artifacts.$kind.bytes-or(@($old.names.$kind)-join"`n")-cne(@($p.names.$kind)-join"`n")){throw 'Original artifact/test tuple'}}}
|
|
if($p.action-eq'read'-and!(Test-Path -LiteralPath $case)){@{id=$p.id;status='remote_case_absent'}|ConvertTo-Json -Compress;exit 0}
|
|
Safe $root $true
|
|
if($p.action-eq'prepare'){if(Test-Path -LiteralPath $case){throw 'Existing case retained'};New-Item -ItemType Directory -Path $case|Out-Null;Safe $case $true;WriteNew (Join-Path $case 'intent.json') $p;@{id=$p.id;prepared=$true}|ConvertTo-Json -Compress;exit 0}
|
|
Safe $case $true;$intent=Join-Path $case 'intent.json';Safe $intent $false;MatchIntent (Get-Content -LiteralPath $intent -Raw|ConvertFrom-Json)
|
|
if($p.action-eq'read'){$result=Join-Path $case 'result.json';if(!(Test-Path -LiteralPath $result)){$status=if(Test-Path -LiteralPath (Join-Path $case 'started.json')){'started_without_result'}else{'prepared_without_start'};@{id=$p.id;status=$status}|ConvertTo-Json -Compress;exit 0};Safe $result $false;Get-Content -LiteralPath $result -Raw;exit 0}
|
|
function RunUnit($exe,[string]$arguments){$s=New-Object Diagnostics.ProcessStartInfo;$s.FileName=$exe;$s.Arguments=$arguments;$s.UseShellExecute=$false;$s.CreateNoWindow=$true;$s.WorkingDirectory=$case;$s.RedirectStandardOutput=$true;$s.RedirectStandardError=$true;$systemRoot=$env:WINDIR;$s.EnvironmentVariables.Clear();$s.EnvironmentVariables['SystemRoot']=$systemRoot;$s.EnvironmentVariables['WINDIR']=$systemRoot;$s.EnvironmentVariables['PATH']=$systemRoot+'\System32';$s.EnvironmentVariables['TEMP']=$script:temp;$s.EnvironmentVariables['TMP']=$script:temp;$s.EnvironmentVariables['RUST_BACKTRACE']='0';$c=New-Object Diagnostics.Process;$c.StartInfo=$s;if(!$c.Start()){throw 'Unit start'};try{$o=$c.StandardOutput.ReadToEndAsync();$e=$c.StandardError.ReadToEndAsync();$timed=$false;if(!$c.WaitForExit(18000)){$timed=$true;$c.Kill();if(!$c.WaitForExit(2000)){throw 'Own unit process did not exit'}};if(!$o.Wait(2000)-or!$e.Wait(2000)-or$o.Result.Length-gt1048576-or$e.Result.Length-gt1048576){throw 'Bounded unit output'};return @{exit_code=$c.ExitCode;timed_out=$timed;stdout_b64=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($o.Result));stderr_b64=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($e.Result))}}finally{$c.Dispose()}}
|
|
WriteNew (Join-Path $case 'started.json') @{execute_once=$true;manifest_sha256=$p.manifest_sha256};$script:temp=Join-Path $case 'tmp';New-Item -ItemType Directory -Path $script:temp|Out-Null;Safe $script:temp $true
|
|
$lists=@{};$executables=@{}
|
|
foreach($kind in @('domain','contracts')){$item=$p.artifacts.$kind;if($item.sha256-notmatch'^[a-f0-9]{64}$'){throw 'Fixed unit hash'};$exe=Join-Path $case ($kind+'.exe');Safe $exe $false;if((Get-FileHash -LiteralPath $exe -Algorithm SHA256).Hash.ToLowerInvariant()-cne$item.sha256-or(Get-Item -LiteralPath $exe).Length-ne$item.bytes){throw 'Exact unit bytes'};$list=RunUnit $exe '--list --format terse';if($list.exit_code-ne0-or$list.timed_out){throw 'Actual Windows listing'};$text=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($list.stdout_b64));$unexpected=@($text-split'\r?\n'|Where-Object{$_.Trim()-and$_-notmatch'^[A-Za-z0-9_:]+: test$'});if($unexpected.Count){throw 'Only exact terse test entries'};$actual=@($text-split'\r?\n'|Where-Object{$_-match': test$'}|ForEach-Object{$_-replace': test$',''}|Sort-Object);$count=if($kind-eq'domain'){16}else{6};if($actual.Count-ne$count-or($actual-join"`n")-cne(@($p.names.$kind)-join"`n")){throw 'All exact names BEFORE any tests'};$lists[$kind]=$actual;$executables[$kind]=$exe}
|
|
$reports=@{};foreach($kind in @('domain','contracts')){$value=RunUnit $executables[$kind] '--test-threads=1';$value.sha256=$p.artifacts.$kind.sha256;$value.names=$lists[$kind];WriteNew (Join-Path $case ($kind+'-result.json')) $value;$reports[$kind]=$value}
|
|
$value=@{id=$p.id;platform='windows';profile=$p.profile;manifest_sha256=$p.manifest_sha256;runner_sha256=$p.runner_sha256;scripts=$p.scripts;linux_proof_sha256=$p.linux_proof_sha256;reports=$reports;bootstrap_administrator_units=$true;local_service_proof=$false;environment_cleared=$true;product_state_writes=$false;rmm_jobs=$false};WriteNew (Join-Path $case 'result.json') $value;$value|ConvertTo-Json -Depth 16 -Compress
|
|
'''
|
|
|
|
|
|
def check_result(manifest, platform, value, payload):
|
|
require(value.get('id') == payload['id'] and value.get('platform') == platform and value.get('profile') == PROFILE
|
|
and value.get('manifest_sha256') == payload['manifest_sha256'] and value.get('runner_sha256') == manifest['runner_sha256']
|
|
and value.get('scripts') == manifest['scripts'] and value.get('product_state_writes') is False and value.get('rmm_jobs') is False
|
|
and value.get('environment_cleared') is True and set(value['reports']) == set(KINDS), 'complete native original binding')
|
|
if platform == 'linux': require(value.get('network_namespace') is True, 'actual Linux netns proof')
|
|
else: require(value.get('linux_proof_sha256') == payload['linux_proof_sha256'] and value.get('local_service_proof') is False, 'original Linux proof and honest account scope')
|
|
for kind, item in value['reports'].items():
|
|
require(item['names'] == EXPECTED[platform][kind] and item['sha256'] == manifest['artifacts'][platform][kind]['sha256']
|
|
and item['exit_code'] == 0 and item['timed_out'] is False, 'all native exact test names/artifacts/exits')
|
|
raw = base64.b64decode(item['stdout_b64'], validate=True); errors = base64.b64decode(item['stderr_b64'], validate=True)
|
|
require(len(raw) <= 1048576 and len(errors) <= 1048576, 'bounded native outputs')
|
|
check_output(raw, COUNTS[platform][kind])
|
|
|
|
|
|
def execute(path, platform, access_root, case_id=None):
|
|
setup_state(); path = path.absolute(); manifest = verified(path, new_run=case_id is None)
|
|
prefix = access_settings(access_root, platform)
|
|
if case_id is None:
|
|
case_id = str(uuid.uuid4()); folder = STATE / case_id; folder.mkdir(mode=0o700)
|
|
payload = dict(id=case_id, profile=PROFILE, manifest_sha256=sha(path), runner_sha256=manifest['runner_sha256'], scripts=manifest['scripts'],
|
|
artifacts={kind: {k: manifest['artifacts'][platform][kind][k] for k in ('sha256', 'bytes')} for kind in KINDS}, names=EXPECTED[platform])
|
|
if platform == 'windows':
|
|
proof_path = path.parent / 'linux-passed.json'; proof = load(proof_path)
|
|
require(proof['verified'] is True and proof['manifest_sha256'] == sha(path), 'same-manifest real Linux proof FIRST')
|
|
linux_payload = dict(payload, id=proof['case_id'], artifacts={kind: {k: manifest['artifacts']['linux'][kind][k] for k in ('sha256', 'bytes')} for kind in KINDS}, names=EXPECTED['linux'])
|
|
check_result(manifest, 'linux', proof['native'], linux_payload)
|
|
payload['linux_proof_sha256'] = sha(proof_path)
|
|
intent = dict(id=case_id, platform=platform, manifest_path=str(path), manifest_sha256=sha(path), runner_sha256=sha(__file__),
|
|
scripts=scripts(), access_root_sha256=hashlib.sha256(str(access_root.absolute()).encode()).hexdigest(), payload=payload)
|
|
save(folder / 'intent.json', intent)
|
|
save(folder / 'started-local.json', dict(create_once=True, no_retry=True))
|
|
run_read = False
|
|
else:
|
|
require(str(uuid.UUID(case_id)) == case_id, 'original canonical case id')
|
|
folder = STATE / case_id; private(folder, True); intent = load(folder / 'intent.json'); load(folder / 'started-local.json')
|
|
require(intent['id'] == case_id and intent['platform'] == platform and intent['manifest_path'] == str(path)
|
|
and intent['manifest_sha256'] == sha(path) and intent['runner_sha256'] == sha(__file__) and intent['scripts'] == scripts()
|
|
and intent['access_root_sha256'] == hashlib.sha256(str(access_root.absolute()).encode()).hexdigest(), 'immutable same-case readback')
|
|
payload = intent['payload']; run_read = True
|
|
rpc = linux_rpc if platform == 'linux' else windows_rpc
|
|
if not run_read:
|
|
result = rpc(prefix, dict(payload, action='prepare'), folder)
|
|
save(folder / 'prepared-remote.json', result); require(result == dict(id=case_id, prepared=True), 'fresh fixed case prepared')
|
|
for kind in KINDS:
|
|
destination = LINUX_REMOTE + '/' + case_id + '/' + kind if platform == 'linux' else '/C:/ProgramData/RustDiscoveryBootstrap/windows-profile-' + case_id + '/' + kind + '.exe'
|
|
upload(prefix, manifest['artifacts'][platform][kind], destination, folder, kind)
|
|
result = rpc(prefix, dict(payload, action='read' if run_read else 'run'), folder)
|
|
save(folder / ('readback-' + str(uuid.uuid4()) + '.json'), result)
|
|
if result.get('status'):
|
|
require(run_read and result.get('id') == case_id and result['status'] in ('remote_case_absent', 'prepared_without_start', 'started_without_result'), 'fixed read-only incomplete status')
|
|
print(json.dumps(dict(verified=False, platform=platform, case_id=case_id, status=result['status'], retry_allowed=False))); return
|
|
check_result(manifest, platform, result, payload)
|
|
proof = dict(verified=True, case_id=case_id, platform=platform, manifest_sha256=sha(path), runner_sha256=manifest['runner_sha256'], native=result)
|
|
destination = path.parent / (platform + '-passed.json')
|
|
if destination.exists(): require(load(destination) == proof, 'preserve identical native proof')
|
|
else: save(destination, proof)
|
|
save(folder / 'verified.json', dict(verified=True, proof_sha256=sha(destination))) if not (folder / 'verified.json').exists() else require(load(folder / 'verified.json') == dict(verified=True, proof_sha256=sha(destination)), 'same verified receipt')
|
|
print(json.dumps(dict(verified=True, platform=platform, case_id=case_id, tests=sum(COUNTS[platform].values()),
|
|
manifest_sha256=sha(path), proof_sha256=sha(destination))))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('build-local', 'linux', 'windows', 'readback-linux', 'readback-windows'))
|
|
parser.add_argument('--manifest', type=Path); parser.add_argument('--case'); parser.add_argument('--access-root', type=Path)
|
|
parser.add_argument('--execute-native-tests', action='store_true'); args = parser.parse_args()
|
|
if args.action == 'build-local':
|
|
require(not (args.manifest or args.case or args.access_root or args.execute_native_tests), 'local build only'); build()
|
|
elif args.action.startswith('readback-'):
|
|
require(args.manifest and args.case and args.access_root and not args.execute_native_tests, 'original case readback only')
|
|
execute(args.manifest, args.action.removeprefix('readback-'), args.access_root, args.case)
|
|
else:
|
|
require(args.manifest and args.access_root and args.execute_native_tests and not args.case, 'explicit new native platform run')
|
|
execute(args.manifest, args.action, args.access_root)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try: main()
|
|
except Exception as error:
|
|
print(json.dumps(dict(error='WINDOWS_PROFILE_QUALIFICATION_INCOMPLETE', error_class=type(error).__name__, details_suppressed=True)))
|
|
raise SystemExit(2) from None
|