314 lines
19 KiB
Python
314 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Native Linux-first qualification of the pure updater verifier, never Apply.
|
|
|
|
prepare-local packages the already-qualified test executable without rebuilding.
|
|
Only the parent orchestrator runs linux/readback-linux, explicitly. No daemon,
|
|
product configuration, enrollment, key, policy, database or RMM job is touched.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import uuid
|
|
|
|
import windows_msi_contract_acceptance as common
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
STATE = ROOT / 'devstate/agent-updater-native'
|
|
PROFILE = 'agent-updater-pure-contracts-v1'
|
|
CORE_COMMIT = 'cc5c93596d0864c526b69a4f7efca805599b6ea1'
|
|
IMAGE = 'sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6'
|
|
REMOTE = '/var/lib/rmm-agent-updater-contracts'
|
|
ARTIFACT_SHA = 'e3a1a3140da09de2f86055faa45faf7ba05ddddc2d92d23fef3df03a63bf2999'
|
|
ARTIFACT_BYTES = 1544584
|
|
EXPECTED = sorted([
|
|
'canonical_release_and_exact_role_bytes_are_metadata_only',
|
|
'signature_domain_bytes_unknown_key_and_revocation_are_independent',
|
|
'signed_noncanonical_unknown_duplicate_and_oversize_payloads_are_rejected',
|
|
'incompatible_clock_platform_state_protocol_and_trust_contexts_are_rejected',
|
|
'monotone_floor_same_sequence_conflict_and_postrollback_state_are_preserved',
|
|
'descriptor_roles_ids_bounds_and_digest_encoding_are_strict',
|
|
'artifact_eof_extra_bytes_hash_and_io_errors_never_verify',
|
|
'streamed_large_release_verifies_with_fixed_read_window',
|
|
'keyring_rejects_weak_keys_duplicates_empty_sets_and_invalid_ranges',
|
|
])
|
|
require = common.require
|
|
save = common.save
|
|
load = common.load
|
|
private = common.private
|
|
sha = common.sha
|
|
|
|
|
|
def setup():
|
|
os.umask(0o077)
|
|
STATE.mkdir(mode=0o700, exist_ok=True)
|
|
private(STATE, True)
|
|
|
|
|
|
def sources():
|
|
"""Only actual core/compiler inputs; later docs/runner commits do not rebuild."""
|
|
names = {'Cargo.toml', 'Cargo.lock'}
|
|
names.update(str(p.relative_to(ROOT)) for p in (ROOT / 'crates/updater').rglob('*') if p.is_file())
|
|
result = {}
|
|
for name in sorted(names):
|
|
path = ROOT / name
|
|
require(path.is_file() and not path.is_symlink(), 'fixed regular core source')
|
|
committed = subprocess.run(['git', 'show', CORE_COMMIT + ':' + name], cwd=ROOT, capture_output=True, check=True).stdout
|
|
require(hashlib.sha256(committed).hexdigest() == sha(path), 'original reviewed core source')
|
|
result[name] = sha(path)
|
|
return result
|
|
|
|
|
|
def driver_sources():
|
|
return {name: sha(ROOT / name) for name in (
|
|
'deploy/agent_updater_contract_acceptance.py',
|
|
'deploy/windows_msi_contract_acceptance.py',
|
|
'deploy/vm_access_check.py',
|
|
)}
|
|
|
|
|
|
def remote_source_sha():
|
|
return hashlib.sha256(inspect.getsource(linux_case).encode()).hexdigest()
|
|
|
|
|
|
def prepare(qualification):
|
|
setup()
|
|
require(qualification.absolute() == ROOT / 'devstate/agent-updater/local-qualification.json', 'existing fixed local qualification')
|
|
old = load(qualification)
|
|
expected_sources = sources()
|
|
require(old['profile'] == PROFILE and old['image_id'] == IMAGE and old['target'] == 'x86_64-unknown-linux-gnu'
|
|
and old['sources'] == expected_sources and old['test_count'] == 9 and old['tests'] == EXPECTED
|
|
and old['native_cloud_vm_tested'] is False and old['agent_started'] is False and old['activation_implemented'] is False,
|
|
'original nine local checks only')
|
|
for label in ('fmt', 'clippy', 'tests'):
|
|
check = old['checks'][label]
|
|
require(check['exit_code'] == 0 and check['log_sha256'] == sha(qualification.parent / (label + '.log'))
|
|
and check['command'][0] == 'docker' and IMAGE in check['command'], 'original local command receipt')
|
|
common.check_output((qualification.parent / 'tests.log').read_bytes(), 9)
|
|
item = old['artifact']
|
|
original = Path(item['path'])
|
|
expected = ROOT / 'devstate/agent-updater/target/x86_64-unknown-linux-gnu/release/deps/release_contract-2aa0a51dd0c5c97d'
|
|
meta = original.lstat()
|
|
require(original == expected and stat.S_ISREG(meta.st_mode) and meta.st_nlink == 1 and meta.st_uid in (0, os.getuid())
|
|
and item['sha256'] == sha(original) == ARTIFACT_SHA and item['bytes'] == meta.st_size == ARTIFACT_BYTES,
|
|
'original qualified link, never rebuilt')
|
|
folder = STATE / ('prepared-' + str(uuid.uuid4()))
|
|
folder.mkdir(mode=0o700)
|
|
artifact = folder / 'updater-contracts'
|
|
fd = os.open(artifact, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'wb') as output:
|
|
output.write(original.read_bytes()); output.flush(); os.fsync(output.fileno())
|
|
require(sha(artifact) == ARTIFACT_SHA and artifact.read_bytes()[:6] == b'\x7fELF\x02\x01'
|
|
and artifact.read_bytes()[18:20] == b'\x3e\0', 'same Linux x64 delivery bytes')
|
|
def inspect_elf(*args):
|
|
return common.command(['docker', 'run', '--rm', '--network', 'none', '--read-only',
|
|
'-v', str(artifact) + ':/unit:ro', '--entrypoint', 'readelf', IMAGE, *args, '/unit'], folder).decode()
|
|
versions = inspect_elf('--version-info')
|
|
dynamic = inspect_elf('-d')
|
|
glibc = sorted(set(re.findall(r'GLIBC_(\d+\.\d+)', versions)), key=lambda n: tuple(map(int, n.split('.'))))
|
|
require(glibc and tuple(map(int, glibc[-1].split('.'))) <= (2, 34), 'known Cloud Linux GLIBC compatibility')
|
|
manifest = dict(schema_version=1, profile=PROFILE, core_commit=CORE_COMMIT,
|
|
local_qualification_path=str(qualification.absolute()), local_qualification_sha256=sha(qualification),
|
|
core_sources=expected_sources, driver_sources=driver_sources(), remote_source_sha256=remote_source_sha(),
|
|
artifact=dict(path=str(artifact), sha256=sha(artifact), bytes=artifact.stat().st_size,
|
|
glibc_max=glibc[-1], needed=sorted(re.findall(r'Shared library: \[([^\]]+)\]', dynamic))),
|
|
test_names=EXPECTED, test_count=9, test_executable_rebuilt=False,
|
|
local_only=True, native_cloud_vm_tested=False, agent_started=False, activation_implemented=False)
|
|
save(folder / 'manifest.json', manifest)
|
|
print(json.dumps(dict(manifest=str(folder / 'manifest.json'), manifest_sha256=sha(folder / 'manifest.json'),
|
|
tests=9, rebuilt=False, native_executed=False)))
|
|
|
|
|
|
def verified(path, new_case):
|
|
path = path.absolute()
|
|
require(path.name == 'manifest.json' and path.parent.parent == STATE
|
|
and re.fullmatch(r'prepared-[0-9a-f-]{36}', path.parent.name), 'fixed native manifest path')
|
|
private(path.parent, True)
|
|
value = load(path)
|
|
require(value['schema_version'] == 1 and value['profile'] == PROFILE and value['core_commit'] == CORE_COMMIT
|
|
and value['driver_sources'] == driver_sources() and value['remote_source_sha256'] == remote_source_sha()
|
|
and value['test_names'] == EXPECTED and value['test_count'] == 9
|
|
and value['test_executable_rebuilt'] is False and value['native_cloud_vm_tested'] is False,
|
|
'original reviewed core and native driver')
|
|
if new_case:
|
|
require(value['core_sources'] == sources(), 'new native case needs same core')
|
|
item = value['artifact']
|
|
require(Path(item['path']) == path.parent / 'updater-contracts'
|
|
and item['sha256'] == ARTIFACT_SHA and item['bytes'] == ARTIFACT_BYTES, 'fixed delivery tuple')
|
|
private(item['path'])
|
|
require(sha(item['path']) == ARTIFACT_SHA and Path(item['path']).stat().st_size == ARTIFACT_BYTES,
|
|
'retained immutable delivery')
|
|
return value
|
|
|
|
|
|
def linux_case(p):
|
|
"""Sent only by the parent to the fixed Cloud Linux bootstrap endpoint."""
|
|
import resource
|
|
ident = str(uuid.UUID(p['id']))
|
|
require(ident == p['id'] and p['profile'] == 'agent-updater-pure-contracts-v1'
|
|
and p['action'] in ('prepare', 'run', 'read') and p['test_count'] == 9 and len(p['names']) == 9,
|
|
'fixed pure updater case')
|
|
require(os.geteuid() == 0 and os.uname().machine == 'x86_64', 'fixed Cloud Linux bootstrap architecture')
|
|
root = Path('/var/lib/rmm-agent-updater-contracts')
|
|
def safe(path, directory, executable=False):
|
|
item = path.lstat()
|
|
require(item.st_uid == 0 and stat.S_IMODE(item.st_mode) == (0o700 if directory else 0o500 if executable else 0o600)
|
|
and (stat.S_ISDIR(item.st_mode) if directory else stat.S_ISREG(item.st_mode) and item.st_nlink == 1), 'private unit object')
|
|
def write(path, value):
|
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'w') as output:
|
|
json.dump(value, output); output.flush(); os.fsync(output.fileno())
|
|
directory = os.open(path.parent, os.O_DIRECTORY | os.O_RDONLY)
|
|
try: os.fsync(directory)
|
|
finally: os.close(directory)
|
|
def read(path):
|
|
safe(path, False); require(path.stat().st_size <= 2 * 1024 * 1024, 'bounded original unit receipt')
|
|
return json.loads(path.read_bytes())
|
|
os.umask(0o077)
|
|
case = root / ident
|
|
if p['action'] == 'read' and not case.exists():
|
|
return dict(id=ident, status='remote_case_absent')
|
|
if p['action'] == 'prepare':
|
|
if not root.exists(): root.mkdir(mode=0o700)
|
|
safe(root, True); case.mkdir(mode=0o700); write(case / 'intent.json', p)
|
|
return dict(id=ident, prepared=True)
|
|
safe(root, True); safe(case, True)
|
|
original = read(case / 'intent.json')
|
|
require({k:v for k,v in p.items() if k != 'action'} == {k:v for k,v in original.items() if k != 'action'}, 'exact original unit intent')
|
|
if p['action'] == 'read':
|
|
if (case / 'result.json').exists(): return read(case / 'result.json')
|
|
partial = {name:read(case / name) for name in ('listing.json',) if (case / name).exists()}
|
|
return dict(id=ident, status='started_without_result' if (case / 'started.json').exists() else 'prepared_without_start', partial=partial)
|
|
write(case / 'started.json', dict(execute_once=True, manifest_sha256=p['manifest_sha256']))
|
|
binary = case / 'updater-contracts'; safe(binary, False)
|
|
require(binary.stat().st_size == p['artifact']['bytes'] == 1544584
|
|
and hashlib.sha256(binary.read_bytes()).hexdigest() == p['artifact']['sha256'] == 'e3a1a3140da09de2f86055faa45faf7ba05ddddc2d92d23fef3df03a63bf2999', 'exact reviewed updater test executable')
|
|
binary.chmod(0o500); safe(binary, False, True)
|
|
temporary = case / 'tmp'; temporary.mkdir(mode=0o700)
|
|
env = {'PATH':'/usr/bin:/bin', 'LANG':'C.UTF-8', 'LC_ALL':'C.UTF-8', 'TMPDIR':str(temporary), 'RUST_BACKTRACE':'0'}
|
|
def limits():
|
|
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
|
resource.setrlimit(resource.RLIMIT_CPU, (15, 15))
|
|
resource.setrlimit(resource.RLIMIT_FSIZE, (1048576, 1048576))
|
|
resource.setrlimit(resource.RLIMIT_AS, (256 * 1024 * 1024, 256 * 1024 * 1024))
|
|
require(subprocess.run(['/usr/bin/unshare','--net','--','/usr/bin/true'], env=env, capture_output=True, timeout=3).returncode == 0, 'isolated network namespace required')
|
|
listed = subprocess.run(['/usr/bin/unshare','--net','--',str(binary),'--list','--format','terse'], cwd=case,
|
|
env=env, capture_output=True, timeout=5, preexec_fn=limits)
|
|
require(listed.returncode == 0 and len(listed.stdout) <= 65536 and not listed.stderr, 'bounded actual test list')
|
|
lines = listed.stdout.decode().splitlines()
|
|
require(all(not line.strip() or __import__('re').fullmatch(r'[A-Za-z0-9_:]+: test', line) for line in lines), 'terse fixed test names only')
|
|
actual = sorted(line[:-6] for line in lines if line.endswith(': test'))
|
|
write(case / 'listing.json', dict(names=actual, sha256=p['artifact']['sha256'], exit_code=listed.returncode))
|
|
require(actual == p['names'] and len(set(actual)) == 9, 'all nine exact names before test execution')
|
|
write(case / 'tests-started.json', dict(execute_once=True))
|
|
with (case / 'stdout.bin').open('xb') as output, (case / 'stderr.bin').open('xb') as errors:
|
|
timed = False
|
|
try:
|
|
code = subprocess.run(['/usr/bin/unshare','--net','--',str(binary),'--test-threads=1'], cwd=case, env=env,
|
|
stdin=subprocess.DEVNULL, stdout=output, stderr=errors, timeout=20, preexec_fn=limits).returncode
|
|
except subprocess.TimeoutExpired: timed = True; code = None
|
|
output.flush(); errors.flush(); os.fsync(output.fileno()); os.fsync(errors.fileno())
|
|
raw, errors = [(case / name).read_bytes() for name in ('stdout.bin', 'stderr.bin')]
|
|
require(len(raw) <= 1048576 and len(errors) <= 1048576, 'bounded original native output')
|
|
result = dict(id=ident, profile=p['profile'], platform='linux', manifest_sha256=p['manifest_sha256'],
|
|
runner_sha256=p['runner_sha256'], remote_source_sha256=p['remote_source_sha256'], core_commit=p['core_commit'],
|
|
artifact=p['artifact'], names=actual, exit_code=code, timed_out=timed,
|
|
stdout_b64=base64.b64encode(raw).decode(), stderr_b64=base64.b64encode(errors).decode(),
|
|
network_namespace=True, environment_cleared=True, list_before_run=True,
|
|
product_state_writes=False, rmm_jobs=False, activation_implemented=False)
|
|
write(case / 'result.json', result)
|
|
return result
|
|
|
|
|
|
def rpc(prefix, payload, folder):
|
|
code = 'import os,json,subprocess,base64,hashlib,uuid\nfrom pathlib import Path\nimport stat\n'
|
|
code += inspect.getsource(common.require) + '\n' + inspect.getsource(linux_case)
|
|
code += '\ntry:\n print(json.dumps(linux_case(' + repr(payload) + ')))\nexcept Exception as e:\n print(json.dumps(dict(bootstrap_error=True,error_class=type(e).__name__)))\n'
|
|
return common.transport(prefix, 'python3 -', code.encode(), folder)
|
|
|
|
|
|
def check_result(value, payload):
|
|
for name in ('id', 'profile', 'manifest_sha256', 'runner_sha256', 'remote_source_sha256', 'core_commit', 'artifact'):
|
|
require(value.get(name) == payload[name], 'complete native original tuple')
|
|
require(value.get('platform') == 'linux' and value.get('names') == EXPECTED and value.get('exit_code') == 0
|
|
and value.get('timed_out') is False and value.get('product_state_writes') is False and value.get('rmm_jobs') is False
|
|
and value.get('activation_implemented') is False and value.get('network_namespace') is True
|
|
and value.get('environment_cleared') is True and value.get('list_before_run') is True, 'all pure native constraints')
|
|
raw = base64.b64decode(value['stdout_b64'], validate=True)
|
|
errors = base64.b64decode(value['stderr_b64'], validate=True)
|
|
require(len(raw) <= 1048576 and not errors, 'bounded clean native output')
|
|
common.check_output(raw, 9)
|
|
|
|
|
|
def execute(path, access_root, case_id):
|
|
is_readback = case_id is not None
|
|
setup(); path = path.absolute(); manifest = verified(path, new_case=not is_readback)
|
|
# This reads access configuration only when the parent explicitly runs native.
|
|
prefix = common.access_settings(access_root, 'linux')
|
|
if case_id is None:
|
|
case_id = str(uuid.uuid4()); folder = STATE / case_id; folder.mkdir(mode=0o700)
|
|
payload = dict(id=case_id, profile=PROFILE, manifest_sha256=sha(path), runner_sha256=sha(__file__),
|
|
remote_source_sha256=remote_source_sha(), core_commit=CORE_COMMIT, test_count=9, names=EXPECTED,
|
|
artifact={name:manifest['artifact'][name] for name in ('sha256', 'bytes')})
|
|
save(folder / 'intent.json', dict(payload=payload, manifest_path=str(path), access_root=str(access_root)))
|
|
save(folder / 'started-local.json', dict(no_retry=True))
|
|
else:
|
|
require(str(uuid.UUID(case_id)) == case_id, 'original canonical case')
|
|
folder = STATE / case_id; private(folder, True)
|
|
original = load(folder / 'intent.json'); load(folder / 'started-local.json')
|
|
require(original['manifest_path'] == str(path) and original['access_root'] == str(access_root), 'same-case access and manifest')
|
|
payload = original['payload']
|
|
require(payload['id'] == case_id and payload['runner_sha256'] == sha(__file__)
|
|
and payload['manifest_sha256'] == sha(path) and payload['remote_source_sha256'] == remote_source_sha(), 'unchanged same-case runner')
|
|
if not case_id or not (folder / 'intent.json').exists():
|
|
raise RuntimeError('missing create-once local intent')
|
|
# Readback never resumes upload or run, even after a lost prepare reply.
|
|
if not is_readback:
|
|
prepared = rpc(prefix, dict(payload, action='prepare'), folder)
|
|
save(folder / 'prepared-remote.json', prepared)
|
|
require(prepared == dict(id=case_id, prepared=True), 'fresh fixed remote scratch case')
|
|
common.upload(prefix, manifest['artifact'], REMOTE + '/' + case_id + '/updater-contracts', folder, 'updater-contracts')
|
|
result = rpc(prefix, dict(payload, action='read' if is_readback else 'run'), folder)
|
|
save(folder / ('readback-' + str(uuid.uuid4()) + '.json'), result)
|
|
if result.get('status'):
|
|
require(is_readback and result.get('id') == case_id and result['status'] in ('remote_case_absent','prepared_without_start','started_without_result'), 'read-only original partial status')
|
|
print(json.dumps(dict(verified=False, case_id=case_id, status=result['status'], retry_allowed=False))); return
|
|
check_result(result, payload)
|
|
proof = dict(verified=True, core_commit=CORE_COMMIT, manifest_sha256=sha(path), case_id=case_id, native=result)
|
|
proof_path = path.parent / 'linux-passed.json'
|
|
if proof_path.exists(): require(load(proof_path) == proof, 'preserve original native proof')
|
|
else: save(proof_path, proof)
|
|
print(json.dumps(dict(verified=True, case_id=case_id, tests=9, proof=str(proof_path), proof_sha256=sha(proof_path), activation_tested=False)))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('prepare-local','linux','readback-linux'))
|
|
parser.add_argument('--qualification', type=Path)
|
|
parser.add_argument('--manifest', type=Path)
|
|
parser.add_argument('--case')
|
|
parser.add_argument('--access-root', type=Path)
|
|
parser.add_argument('--execute-native-tests', action='store_true')
|
|
args = parser.parse_args()
|
|
if args.action == 'prepare-local':
|
|
require(args.qualification and not (args.manifest or args.case or args.access_root or args.execute_native_tests), 'prepare packages only existing local unit artifact')
|
|
prepare(args.qualification)
|
|
else:
|
|
require(args.manifest and args.access_root and not args.qualification, 'explicit original manifest and access root')
|
|
if args.action == 'linux': require(args.execute_native_tests and not args.case, 'explicit new native unit invocation')
|
|
else: require(args.case and not args.execute_native_tests, 'readback never executes tests')
|
|
execute(args.manifest, args.access_root, args.case)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try: main()
|
|
except Exception as error:
|
|
print(json.dumps(dict(error='AGENT_UPDATER_NATIVE_QUALIFICATION_INCOMPLETE', error_class=type(error).__name__, details_suppressed=True)))
|
|
raise SystemExit(2) from None
|