280 lines
16 KiB
Python
280 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Package existing five portable clock units; only Root invokes native phases.
|
|
|
|
No build, Windows/NDR execution, RMM daemon, service, policy, DB or job action.
|
|
Lost native replies permit readback of the same case only, never another run.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'deploy'))
|
|
import remote_client_bootstrap as linux
|
|
import vm_mvp_acceptance as contract
|
|
|
|
CASE = ROOT / 'devstate/windows-clock-midl/diagnostic-builds/181c3a1c-8324-41fd-9b7c-aa8010262912'
|
|
QUALIFICATION_SHA = '2e0de3acf72ab5c3a8e0baa54e8643d086ea7c03d1b38fd89b6bfc00e45f8cbb'
|
|
STATE = ROOT / 'devstate/windows-clock-midl/linux-native-packs'
|
|
REMOTE = '/var/lib/rmm-clock-midl-portable-tests'
|
|
TESTS = (
|
|
'clock::tests::gate_requires_all_status_and_uncertainty_evidence',
|
|
'clock::tests::production_clock_stays_closed_until_native_adapter_qualification',
|
|
'clock::tests::suspend_closes_until_actual_post_resume_sync',
|
|
'clock::tests::timeout_worker_stays_closed_without_a_second_call',
|
|
'clock::tests::wall_step_and_overflow_fail_closed',
|
|
)
|
|
require = contract.require
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
|
|
|
|
def support_sources():
|
|
return {str(Path(module.__file__).resolve().relative_to(ROOT)): sha(module.__file__)
|
|
for module in (linux, contract, linux.access)}
|
|
|
|
|
|
def native_source_sha256():
|
|
return hashlib.sha256(inspect.getsource(native_case).encode()).hexdigest()
|
|
|
|
|
|
def save(folder, name, value):
|
|
fd = os.open(folder / name, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'w') as out:
|
|
json.dump(value, out, indent=2)
|
|
out.write('\n')
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
|
|
|
|
def prepare():
|
|
require(sha(CASE / 'qualification.json') == QUALIFICATION_SHA, 'original build qualification')
|
|
qualified = contract.protected_json(CASE / 'qualification.json')
|
|
require(qualified['linux_clock_test_count'] == 5 and qualified['no_vm'] is True, 'five local portable guards only')
|
|
original = CASE / 'target/debug/deps/clock_midl_readiness-277aee462cb4f674'
|
|
raw = original.read_bytes()
|
|
require(raw[:6] == b'\x7fELF\x02\x01' and raw[18:20] == b'\x3e\x00', 'existing x64 Linux test executable')
|
|
os.umask(0o077)
|
|
STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
folder = STATE / str(uuid.uuid4())
|
|
folder.mkdir(mode=0o700)
|
|
binary = folder / 'clock-portable-tests'
|
|
shutil.copyfile(original, binary)
|
|
binary.chmod(0o600)
|
|
require(sha(binary) == sha(original), 'byte-identical existing test executable')
|
|
for name, args in (
|
|
('elf', ['readelf', '--wide', '--program-headers', '--dynamic', '--version-info', str(binary)]),
|
|
('list', ['docker', 'run', '--rm', '--network', 'none', '--read-only', '-v', str(binary) + ':/tests:ro',
|
|
'--entrypoint', '/lib64/ld-linux-x86-64.so.2', qualified['image'], '/tests', '--list', '--format', 'terse']),
|
|
):
|
|
completed = subprocess.run(args, capture_output=True, timeout=15)
|
|
require(completed.returncode == 0, 'local loader/list metadata failed')
|
|
fd = os.open(folder / (name + '.log'), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
|
with os.fdopen(fd, 'wb') as out:
|
|
out.write(completed.stdout)
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
listed = (folder / 'list.log').read_bytes()
|
|
require(set(re.findall(rb'^(.+): test$', listed, re.M)) == {name.encode() for name in TESTS}, 'exact existing five-test list')
|
|
info = (folder / 'elf.log').read_text()
|
|
needed = re.findall(r'\(NEEDED\).*?\[([^\]]+)\]', info)
|
|
glibc = sorted(set(re.findall(r'GLIBC_[0-9.]+', info)), key=lambda v: tuple(map(int, v[6:].split('.'))))
|
|
require('/lib64/ld-linux-x86-64.so.2' in info and needed and glibc, 'actual ELF loader dependencies')
|
|
manifest = dict(schema_version=1, type='rmm.clock.portable-unit-native-pack', original_build=str(CASE),
|
|
qualification_sha256=QUALIFICATION_SHA, runner_sha256=sha(__file__),
|
|
support_sources=support_sources(), native_case_sha256=native_source_sha256(),
|
|
artifact=dict(path=str(binary), bytes=binary.stat().st_size, sha256=sha(binary)),
|
|
original_test_executable=str(original), selected_tests=list(TESTS),
|
|
interpreter='/lib64/ld-linux-x86-64.so.2', needed=needed, maximum_glibc=glibc[-1],
|
|
list_sha256=sha(folder / 'list.log'), elf_sha256=sha(folder / 'elf.log'),
|
|
compiled_new_code=False, stripped=False, native_vm_executed=False,
|
|
linux_ndr_tested=False, windows_rpc_tested=False, product_clock_qualified=False)
|
|
save(folder, 'manifest.json', manifest)
|
|
print(json.dumps(dict(manifest=str(folder / 'manifest.json'), manifest_sha256=sha(folder / 'manifest.json'),
|
|
artifact=manifest['artifact'], interpreter=manifest['interpreter'], needed=needed,
|
|
maximum_glibc=glibc[-1], selected_tests=5, native_vm_executed=False)))
|
|
|
|
|
|
def verified(path):
|
|
manifest = contract.protected_json(path)
|
|
require(manifest['runner_sha256'] == sha(__file__) and manifest['selected_tests'] == list(TESTS), 'frozen native pack runner and exact tests')
|
|
require(manifest['support_sources'] == support_sources() and manifest['native_case_sha256'] == native_source_sha256(), 'frozen native transport/parser sources')
|
|
require(manifest['qualification_sha256'] == QUALIFICATION_SHA == sha(CASE / 'qualification.json'), 'actual original build')
|
|
require(manifest['compiled_new_code'] is False and manifest['stripped'] is False and manifest['native_vm_executed'] is False, 'existing portable harness only')
|
|
binary = Path(manifest['artifact']['path'])
|
|
require(sha(binary) == manifest['artifact']['sha256'] and binary.stat().st_size == manifest['artifact']['bytes']
|
|
and sha(manifest['original_test_executable']) == sha(binary), 'unchanged actual original Linux harness')
|
|
return manifest
|
|
|
|
|
|
def native_case(payload):
|
|
import resource
|
|
import stat
|
|
import time
|
|
root = Path('/var/lib/rmm-clock-midl-portable-tests')
|
|
ident = str(uuid.UUID(payload['id']))
|
|
names = [
|
|
'clock::tests::gate_requires_all_status_and_uncertainty_evidence',
|
|
'clock::tests::production_clock_stays_closed_until_native_adapter_qualification',
|
|
'clock::tests::suspend_closes_until_actual_post_resume_sync',
|
|
'clock::tests::timeout_worker_stays_closed_without_a_second_call',
|
|
'clock::tests::wall_step_and_overflow_fail_closed',
|
|
]
|
|
require(payload['id'] == ident and payload['action'] in ('prepare', 'run', 'read')
|
|
and payload['tests'] == names and os.geteuid() == 0, 'fixed exact portable test case')
|
|
digest = payload['binary_sha256']
|
|
require(len(digest) == 64 and all(c in '0123456789abcdef' for c in digest), 'fixed binary hash')
|
|
def safe(path, directory):
|
|
value = path.lstat()
|
|
require(value.st_uid == 0 and not value.st_mode & 0o077 and not stat.S_ISLNK(value.st_mode)
|
|
and (stat.S_ISDIR(value.st_mode) if directory else stat.S_ISREG(value.st_mode)), 'private fixed test path')
|
|
def write(path, value):
|
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'w') as out:
|
|
json.dump(value, out)
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
if payload['action'] == 'prepare' and not root.exists():
|
|
root.mkdir(mode=0o700)
|
|
safe(root, True)
|
|
case = root / ident
|
|
if payload['action'] == 'prepare':
|
|
case.mkdir(mode=0o700)
|
|
write(case / 'intent.json', payload)
|
|
return dict(prepared=True)
|
|
safe(case, True)
|
|
safe(case / 'intent.json', False)
|
|
intent = json.loads((case / 'intent.json').read_bytes())
|
|
require(all(intent[key] == payload[key] for key in ('id', 'binary_sha256', 'manifest_sha256', 'tests', 'runner_sha256', 'support_sources', 'native_case_sha256')), 'original create-once intent')
|
|
if payload['action'] == 'read':
|
|
safe(case / 'result.json', False)
|
|
return json.loads((case / 'result.json').read_bytes())
|
|
binary = case / 'clock-portable-tests'
|
|
safe(binary, False)
|
|
require(hashlib.sha256(binary.read_bytes()).hexdigest() == digest, 'actual staged original test bytes')
|
|
write(case / 'started.json', dict(binary_sha256=digest, tests=names, never_repeat_start=True))
|
|
binary.chmod(0o500)
|
|
temporary = case / 'tmp'
|
|
temporary.mkdir(mode=0o700)
|
|
env = {'PATH': '/usr/bin:/bin', 'LANG': 'C.UTF-8', 'TMPDIR': str(temporary), 'RUST_BACKTRACE': '0'}
|
|
namespace = subprocess.run(['/usr/bin/unshare', '--net', '--', '/usr/bin/true'], env=env, capture_output=True, timeout=3)
|
|
require(namespace.returncode == 0, 'network namespace required; no fallback')
|
|
loader = subprocess.run(['/lib64/ld-linux-x86-64.so.2', '--list', str(binary)], env=env, capture_output=True, timeout=3)
|
|
require(loader.returncode == 0 and len(loader.stdout) <= 16384 and len(loader.stderr) <= 16384, 'actual native loader compatibility')
|
|
def limits():
|
|
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
|
resource.setrlimit(resource.RLIMIT_FSIZE, (1048576, 1048576))
|
|
resource.setrlimit(resource.RLIMIT_CPU, (10, 10))
|
|
results = []
|
|
deadline = time.monotonic() + 20
|
|
for number, name in enumerate(names):
|
|
output = case / (str(number) + '-stdout.bin')
|
|
errors = case / (str(number) + '-stderr.bin')
|
|
timed_out = False
|
|
with output.open('xb') as stdout, errors.open('xb') as stderr:
|
|
output.chmod(0o600)
|
|
errors.chmod(0o600)
|
|
try:
|
|
remaining = deadline - time.monotonic()
|
|
require(remaining > 0, 'aggregate test deadline')
|
|
child = subprocess.run(['/usr/bin/unshare', '--net', '--', str(binary), '--exact', name, '--test-threads=1'],
|
|
cwd=case, env=env, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr,
|
|
timeout=min(5, remaining), preexec_fn=limits)
|
|
code = child.returncode
|
|
except subprocess.TimeoutExpired:
|
|
timed_out = True
|
|
code = None
|
|
stdout.flush()
|
|
stderr.flush()
|
|
os.fsync(stdout.fileno())
|
|
os.fsync(stderr.fileno())
|
|
raw, err = output.read_bytes(), errors.read_bytes()
|
|
require(len(raw) <= 1048576 and len(err) <= 1048576, 'bounded unit output')
|
|
results.append(dict(name=name, exit_code=code, timed_out=timed_out, stdout_b64=base64.b64encode(raw).decode(), stderr_b64=base64.b64encode(err).decode()))
|
|
if code != 0:
|
|
break
|
|
report = dict(id=ident, binary_sha256=digest, manifest_sha256=payload['manifest_sha256'], results=results,
|
|
native_platform='linux', network_namespace=True, environment_cleared=True,
|
|
loader_stdout_sha256=hashlib.sha256(loader.stdout).hexdigest(), linux_ndr_tested=False,
|
|
windows_rpc_tested=False, product_clock_qualified=False, rmm_jobs_created=False)
|
|
write(case / 'result.json', report)
|
|
return report
|
|
|
|
|
|
def native(path, access_root, ident=None):
|
|
manifest = verified(path)
|
|
linux.ROOT = linux.access.ROOT = access_root.resolve()
|
|
os.umask(0o077)
|
|
if ident is None:
|
|
ident = str(uuid.uuid4())
|
|
folder = STATE / ident
|
|
folder.mkdir(mode=0o700)
|
|
payload = dict(id=ident, binary_sha256=manifest['artifact']['sha256'], manifest_sha256=sha(path), tests=list(TESTS),
|
|
runner_sha256=manifest['runner_sha256'], support_sources=manifest['support_sources'], native_case_sha256=manifest['native_case_sha256'])
|
|
save(folder, 'intent.json', dict(payload=payload, manifest_path=str(path.resolve())))
|
|
linux.rpc('linux-vm.conf', native_case, dict(payload, action='prepare'))
|
|
cfg = linux.access.load_config(access_root / 'credentials/linux-vm.conf')
|
|
args = ['sftp', '-F', '/dev/null', '-P', cfg.get('port', '22'), '-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes',
|
|
'-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10', '-i', str(Path(cfg['ssh_key_path']).expanduser()),
|
|
'-b', '-', cfg['username'] + '@' + cfg['address']]
|
|
result = subprocess.run(args, input=('put ' + json.dumps(manifest['artifact']['path']) + ' ' + json.dumps(REMOTE + '/' + ident + '/clock-portable-tests') + '\n').encode(), capture_output=True, timeout=120)
|
|
save(folder, 'upload.json', dict(returncode=result.returncode))
|
|
require(result.returncode == 0, 'fixed portable binary upload')
|
|
save(folder, 'run-intent.json', dict(execute_only_exact_five_portable_units=True))
|
|
value = linux.rpc('linux-vm.conf', native_case, dict(payload, action='run'))
|
|
else:
|
|
ident = str(uuid.UUID(ident))
|
|
folder = STATE / ident
|
|
intent = contract.protected_json(folder / 'intent.json')
|
|
payload = intent['payload']
|
|
require(intent['manifest_path'] == str(path.resolve()) and payload['manifest_sha256'] == sha(path), 'original same native case')
|
|
require(payload['runner_sha256'] == manifest['runner_sha256'] and payload['support_sources'] == manifest['support_sources']
|
|
and payload['native_case_sha256'] == manifest['native_case_sha256'], 'original same native sources')
|
|
value = linux.rpc('linux-vm.conf', native_case, dict(payload, action='read'))
|
|
save(folder, 'readback-' + str(uuid.uuid4()) + '.json', value)
|
|
require(value['binary_sha256'] == manifest['artifact']['sha256'] and value['manifest_sha256'] == sha(path)
|
|
and len(value['results']) == 5 and value['product_clock_qualified'] is False, 'all five exact portable results')
|
|
for result, name in zip(value['results'], TESTS, strict=True):
|
|
require(result['name'] == name and result['exit_code'] == 0 and result['timed_out'] is False
|
|
and b'1 passed; 0 failed; 0 ignored;' in base64.b64decode(result['stdout_b64'], validate=True), 'one exact native portable unit passed')
|
|
proof = dict(verified=True, case_id=ident, native_platform='linux', selected_tests=5, manifest_sha256=sha(path), report=value)
|
|
if (folder / 'passed.json').exists():
|
|
require(contract.protected_json(folder / 'passed.json') == proof, 'unchanged original native proof')
|
|
else:
|
|
save(folder, 'passed.json', proof)
|
|
print(json.dumps(dict(verified=True, case_id=ident, selected_native_portable_tests=5, linux_ndr_tested=False, product_clock_qualified=False)))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('prepare-local', 'native', 'readback'))
|
|
parser.add_argument('--manifest', type=Path)
|
|
parser.add_argument('--access-root', type=Path)
|
|
parser.add_argument('--case')
|
|
parser.add_argument('--execute-native-tests', action='store_true')
|
|
args = parser.parse_args()
|
|
if args.action == 'prepare-local':
|
|
require(not (args.manifest or args.access_root or args.case or args.execute_native_tests), 'local packaging only')
|
|
prepare()
|
|
elif args.action == 'native':
|
|
require(args.manifest and args.access_root and args.execute_native_tests and not args.case, 'Root explicitly starts one fixed native case')
|
|
native(args.manifest, args.access_root)
|
|
else:
|
|
require(args.manifest and args.access_root and args.case and not args.execute_native_tests, 'same-case readback only')
|
|
native(args.manifest, args.access_root, args.case)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|