#!/usr/bin/env python3 """Package existing five portable clock units; only Root invokes native phases. No build, Windows/NDR execution, RMM daemon, service, policy, DB or job action. Lost native replies permit readback of the same case only, never another run. """ import argparse import base64 import hashlib import inspect import json import os from pathlib import Path import re import shutil import subprocess import sys import uuid ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'deploy')) import remote_client_bootstrap as linux import vm_mvp_acceptance as contract CASE = ROOT / 'devstate/windows-clock-midl/diagnostic-builds/181c3a1c-8324-41fd-9b7c-aa8010262912' QUALIFICATION_SHA = '2e0de3acf72ab5c3a8e0baa54e8643d086ea7c03d1b38fd89b6bfc00e45f8cbb' STATE = ROOT / 'devstate/windows-clock-midl/linux-native-packs' REMOTE = '/var/lib/rmm-clock-midl-portable-tests' TESTS = ( 'clock::tests::gate_requires_all_status_and_uncertainty_evidence', 'clock::tests::production_clock_stays_closed_until_native_adapter_qualification', 'clock::tests::suspend_closes_until_actual_post_resume_sync', 'clock::tests::timeout_worker_stays_closed_without_a_second_call', 'clock::tests::wall_step_and_overflow_fail_closed', ) require = contract.require def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def support_sources(): return {str(Path(module.__file__).resolve().relative_to(ROOT)): sha(module.__file__) for module in (linux, contract, linux.access)} def native_source_sha256(): return hashlib.sha256(inspect.getsource(native_case).encode()).hexdigest() def save(folder, name, value): fd = os.open(folder / name, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, 'w') as out: json.dump(value, out, indent=2) out.write('\n') out.flush() os.fsync(out.fileno()) def prepare(): require(sha(CASE / 'qualification.json') == QUALIFICATION_SHA, 'original build qualification') qualified = contract.protected_json(CASE / 'qualification.json') require(qualified['linux_clock_test_count'] == 5 and qualified['no_vm'] is True, 'five local portable guards only') original = CASE / 'target/debug/deps/clock_midl_readiness-277aee462cb4f674' raw = original.read_bytes() require(raw[:6] == b'\x7fELF\x02\x01' and raw[18:20] == b'\x3e\x00', 'existing x64 Linux test executable') os.umask(0o077) STATE.mkdir(mode=0o700, parents=True, exist_ok=True) folder = STATE / str(uuid.uuid4()) folder.mkdir(mode=0o700) binary = folder / 'clock-portable-tests' shutil.copyfile(original, binary) binary.chmod(0o600) require(sha(binary) == sha(original), 'byte-identical existing test executable') for name, args in ( ('elf', ['readelf', '--wide', '--program-headers', '--dynamic', '--version-info', str(binary)]), ('list', ['docker', 'run', '--rm', '--network', 'none', '--read-only', '-v', str(binary) + ':/tests:ro', '--entrypoint', '/lib64/ld-linux-x86-64.so.2', qualified['image'], '/tests', '--list', '--format', 'terse']), ): completed = subprocess.run(args, capture_output=True, timeout=15) require(completed.returncode == 0, 'local loader/list metadata failed') fd = os.open(folder / (name + '.log'), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) with os.fdopen(fd, 'wb') as out: out.write(completed.stdout) out.flush() os.fsync(out.fileno()) listed = (folder / 'list.log').read_bytes() require(set(re.findall(rb'^(.+): test$', listed, re.M)) == {name.encode() for name in TESTS}, 'exact existing five-test list') info = (folder / 'elf.log').read_text() needed = re.findall(r'\(NEEDED\).*?\[([^\]]+)\]', info) glibc = sorted(set(re.findall(r'GLIBC_[0-9.]+', info)), key=lambda v: tuple(map(int, v[6:].split('.')))) require('/lib64/ld-linux-x86-64.so.2' in info and needed and glibc, 'actual ELF loader dependencies') manifest = dict(schema_version=1, type='rmm.clock.portable-unit-native-pack', original_build=str(CASE), qualification_sha256=QUALIFICATION_SHA, runner_sha256=sha(__file__), support_sources=support_sources(), native_case_sha256=native_source_sha256(), artifact=dict(path=str(binary), bytes=binary.stat().st_size, sha256=sha(binary)), original_test_executable=str(original), selected_tests=list(TESTS), interpreter='/lib64/ld-linux-x86-64.so.2', needed=needed, maximum_glibc=glibc[-1], list_sha256=sha(folder / 'list.log'), elf_sha256=sha(folder / 'elf.log'), compiled_new_code=False, stripped=False, native_vm_executed=False, linux_ndr_tested=False, windows_rpc_tested=False, product_clock_qualified=False) save(folder, 'manifest.json', manifest) print(json.dumps(dict(manifest=str(folder / 'manifest.json'), manifest_sha256=sha(folder / 'manifest.json'), artifact=manifest['artifact'], interpreter=manifest['interpreter'], needed=needed, maximum_glibc=glibc[-1], selected_tests=5, native_vm_executed=False))) def verified(path): manifest = contract.protected_json(path) require(manifest['runner_sha256'] == sha(__file__) and manifest['selected_tests'] == list(TESTS), 'frozen native pack runner and exact tests') require(manifest['support_sources'] == support_sources() and manifest['native_case_sha256'] == native_source_sha256(), 'frozen native transport/parser sources') require(manifest['qualification_sha256'] == QUALIFICATION_SHA == sha(CASE / 'qualification.json'), 'actual original build') require(manifest['compiled_new_code'] is False and manifest['stripped'] is False and manifest['native_vm_executed'] is False, 'existing portable harness only') binary = Path(manifest['artifact']['path']) require(sha(binary) == manifest['artifact']['sha256'] and binary.stat().st_size == manifest['artifact']['bytes'] and sha(manifest['original_test_executable']) == sha(binary), 'unchanged actual original Linux harness') return manifest def native_case(payload): import resource import stat import time root = Path('/var/lib/rmm-clock-midl-portable-tests') ident = str(uuid.UUID(payload['id'])) names = [ 'clock::tests::gate_requires_all_status_and_uncertainty_evidence', 'clock::tests::production_clock_stays_closed_until_native_adapter_qualification', 'clock::tests::suspend_closes_until_actual_post_resume_sync', 'clock::tests::timeout_worker_stays_closed_without_a_second_call', 'clock::tests::wall_step_and_overflow_fail_closed', ] require(payload['id'] == ident and payload['action'] in ('prepare', 'run', 'read') and payload['tests'] == names and os.geteuid() == 0, 'fixed exact portable test case') digest = payload['binary_sha256'] require(len(digest) == 64 and all(c in '0123456789abcdef' for c in digest), 'fixed binary hash') def safe(path, directory): value = path.lstat() require(value.st_uid == 0 and not value.st_mode & 0o077 and not stat.S_ISLNK(value.st_mode) and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'private fixed test path') def write(path, value): fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, 'w') as out: json.dump(value, out) out.flush() os.fsync(out.fileno()) if payload['action'] == 'prepare' and not root.exists(): root.mkdir(mode=0o700) safe(root, True) case = root / ident if payload['action'] == 'prepare': case.mkdir(mode=0o700) write(case / 'intent.json', payload) return dict(prepared=True) safe(case, True) safe(case / 'intent.json', False) intent = json.loads((case / 'intent.json').read_bytes()) require(all(intent[key] == payload[key] for key in ('id', 'binary_sha256', 'manifest_sha256', 'tests', 'runner_sha256', 'support_sources', 'native_case_sha256')), 'original create-once intent') if payload['action'] == 'read': safe(case / 'result.json', False) return json.loads((case / 'result.json').read_bytes()) binary = case / 'clock-portable-tests' safe(binary, False) require(hashlib.sha256(binary.read_bytes()).hexdigest() == digest, 'actual staged original test bytes') write(case / 'started.json', dict(binary_sha256=digest, tests=names, never_repeat_start=True)) binary.chmod(0o500) temporary = case / 'tmp' temporary.mkdir(mode=0o700) env = {'PATH': '/usr/bin:/bin', 'LANG': 'C.UTF-8', 'TMPDIR': str(temporary), 'RUST_BACKTRACE': '0'} namespace = subprocess.run(['/usr/bin/unshare', '--net', '--', '/usr/bin/true'], env=env, capture_output=True, timeout=3) require(namespace.returncode == 0, 'network namespace required; no fallback') loader = subprocess.run(['/lib64/ld-linux-x86-64.so.2', '--list', str(binary)], env=env, capture_output=True, timeout=3) require(loader.returncode == 0 and len(loader.stdout) <= 16384 and len(loader.stderr) <= 16384, 'actual native loader compatibility') def limits(): resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) resource.setrlimit(resource.RLIMIT_FSIZE, (1048576, 1048576)) resource.setrlimit(resource.RLIMIT_CPU, (10, 10)) results = [] deadline = time.monotonic() + 20 for number, name in enumerate(names): output = case / (str(number) + '-stdout.bin') errors = case / (str(number) + '-stderr.bin') timed_out = False with output.open('xb') as stdout, errors.open('xb') as stderr: output.chmod(0o600) errors.chmod(0o600) try: remaining = deadline - time.monotonic() require(remaining > 0, 'aggregate test deadline') child = subprocess.run(['/usr/bin/unshare', '--net', '--', str(binary), '--exact', name, '--test-threads=1'], cwd=case, env=env, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr, timeout=min(5, remaining), preexec_fn=limits) code = child.returncode except subprocess.TimeoutExpired: timed_out = True code = None stdout.flush() stderr.flush() os.fsync(stdout.fileno()) os.fsync(stderr.fileno()) raw, err = output.read_bytes(), errors.read_bytes() require(len(raw) <= 1048576 and len(err) <= 1048576, 'bounded unit output') results.append(dict(name=name, exit_code=code, timed_out=timed_out, stdout_b64=base64.b64encode(raw).decode(), stderr_b64=base64.b64encode(err).decode())) if code != 0: break report = dict(id=ident, binary_sha256=digest, manifest_sha256=payload['manifest_sha256'], results=results, native_platform='linux', network_namespace=True, environment_cleared=True, loader_stdout_sha256=hashlib.sha256(loader.stdout).hexdigest(), linux_ndr_tested=False, windows_rpc_tested=False, product_clock_qualified=False, rmm_jobs_created=False) write(case / 'result.json', report) return report def native(path, access_root, ident=None): manifest = verified(path) linux.ROOT = linux.access.ROOT = access_root.resolve() os.umask(0o077) if ident is None: ident = str(uuid.uuid4()) folder = STATE / ident folder.mkdir(mode=0o700) payload = dict(id=ident, binary_sha256=manifest['artifact']['sha256'], manifest_sha256=sha(path), tests=list(TESTS), runner_sha256=manifest['runner_sha256'], support_sources=manifest['support_sources'], native_case_sha256=manifest['native_case_sha256']) save(folder, 'intent.json', dict(payload=payload, manifest_path=str(path.resolve()))) linux.rpc('linux-vm.conf', native_case, dict(payload, action='prepare')) cfg = linux.access.load_config(access_root / 'credentials/linux-vm.conf') args = ['sftp', '-F', '/dev/null', '-P', cfg.get('port', '22'), '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10', '-i', str(Path(cfg['ssh_key_path']).expanduser()), '-b', '-', cfg['username'] + '@' + cfg['address']] result = subprocess.run(args, input=('put ' + json.dumps(manifest['artifact']['path']) + ' ' + json.dumps(REMOTE + '/' + ident + '/clock-portable-tests') + '\n').encode(), capture_output=True, timeout=120) save(folder, 'upload.json', dict(returncode=result.returncode)) require(result.returncode == 0, 'fixed portable binary upload') save(folder, 'run-intent.json', dict(execute_only_exact_five_portable_units=True)) value = linux.rpc('linux-vm.conf', native_case, dict(payload, action='run')) else: ident = str(uuid.UUID(ident)) folder = STATE / ident intent = contract.protected_json(folder / 'intent.json') payload = intent['payload'] require(intent['manifest_path'] == str(path.resolve()) and payload['manifest_sha256'] == sha(path), 'original same native case') require(payload['runner_sha256'] == manifest['runner_sha256'] and payload['support_sources'] == manifest['support_sources'] and payload['native_case_sha256'] == manifest['native_case_sha256'], 'original same native sources') value = linux.rpc('linux-vm.conf', native_case, dict(payload, action='read')) save(folder, 'readback-' + str(uuid.uuid4()) + '.json', value) require(value['binary_sha256'] == manifest['artifact']['sha256'] and value['manifest_sha256'] == sha(path) and len(value['results']) == 5 and value['product_clock_qualified'] is False, 'all five exact portable results') for result, name in zip(value['results'], TESTS, strict=True): require(result['name'] == name and result['exit_code'] == 0 and result['timed_out'] is False and b'1 passed; 0 failed; 0 ignored;' in base64.b64decode(result['stdout_b64'], validate=True), 'one exact native portable unit passed') proof = dict(verified=True, case_id=ident, native_platform='linux', selected_tests=5, manifest_sha256=sha(path), report=value) if (folder / 'passed.json').exists(): require(contract.protected_json(folder / 'passed.json') == proof, 'unchanged original native proof') else: save(folder, 'passed.json', proof) print(json.dumps(dict(verified=True, case_id=ident, selected_native_portable_tests=5, linux_ndr_tested=False, product_clock_qualified=False))) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=('prepare-local', 'native', 'readback')) parser.add_argument('--manifest', type=Path) parser.add_argument('--access-root', type=Path) parser.add_argument('--case') parser.add_argument('--execute-native-tests', action='store_true') args = parser.parse_args() if args.action == 'prepare-local': require(not (args.manifest or args.access_root or args.case or args.execute_native_tests), 'local packaging only') prepare() elif args.action == 'native': require(args.manifest and args.access_root and args.execute_native_tests and not args.case, 'Root explicitly starts one fixed native case') native(args.manifest, args.access_root) else: require(args.manifest and args.access_root and args.case and not args.execute_native_tests, 'same-case readback only') native(args.manifest, args.access_root, args.case) if __name__ == '__main__': main()