Merge reviewed default-deny agent updater source checkpoint
# Conflicts: # docs/agent-updater-linux-native-results.md
This commit is contained in:
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Native Linux-first qualification of the pure updater verifier, never Apply.
|
||||
|
||||
prepare-local packages the already-qualified test executable without rebuilding.
|
||||
Only the parent orchestrator runs linux/readback-linux, explicitly. No daemon,
|
||||
product configuration, enrollment, key, policy, database or RMM job is touched.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import uuid
|
||||
|
||||
import windows_msi_contract_acceptance as common
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STATE = ROOT / 'devstate/agent-updater-native'
|
||||
PROFILE = 'agent-updater-pure-contracts-v1'
|
||||
CORE_COMMIT = 'cc5c93596d0864c526b69a4f7efca805599b6ea1'
|
||||
IMAGE = 'sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6'
|
||||
REMOTE = '/var/lib/rmm-agent-updater-contracts'
|
||||
ARTIFACT_SHA = 'e3a1a3140da09de2f86055faa45faf7ba05ddddc2d92d23fef3df03a63bf2999'
|
||||
ARTIFACT_BYTES = 1544584
|
||||
EXPECTED = sorted([
|
||||
'canonical_release_and_exact_role_bytes_are_metadata_only',
|
||||
'signature_domain_bytes_unknown_key_and_revocation_are_independent',
|
||||
'signed_noncanonical_unknown_duplicate_and_oversize_payloads_are_rejected',
|
||||
'incompatible_clock_platform_state_protocol_and_trust_contexts_are_rejected',
|
||||
'monotone_floor_same_sequence_conflict_and_postrollback_state_are_preserved',
|
||||
'descriptor_roles_ids_bounds_and_digest_encoding_are_strict',
|
||||
'artifact_eof_extra_bytes_hash_and_io_errors_never_verify',
|
||||
'streamed_large_release_verifies_with_fixed_read_window',
|
||||
'keyring_rejects_weak_keys_duplicates_empty_sets_and_invalid_ranges',
|
||||
])
|
||||
require = common.require
|
||||
save = common.save
|
||||
load = common.load
|
||||
private = common.private
|
||||
sha = common.sha
|
||||
|
||||
|
||||
def setup():
|
||||
os.umask(0o077)
|
||||
STATE.mkdir(mode=0o700, exist_ok=True)
|
||||
private(STATE, True)
|
||||
|
||||
|
||||
def sources():
|
||||
"""Only actual core/compiler inputs; later docs/runner commits do not rebuild."""
|
||||
names = {'Cargo.toml', 'Cargo.lock'}
|
||||
names.update(str(p.relative_to(ROOT)) for p in (ROOT / 'crates/updater').rglob('*') if p.is_file())
|
||||
result = {}
|
||||
for name in sorted(names):
|
||||
path = ROOT / name
|
||||
require(path.is_file() and not path.is_symlink(), 'fixed regular core source')
|
||||
committed = subprocess.run(['git', 'show', CORE_COMMIT + ':' + name], cwd=ROOT, capture_output=True, check=True).stdout
|
||||
require(hashlib.sha256(committed).hexdigest() == sha(path), 'original reviewed core source')
|
||||
result[name] = sha(path)
|
||||
return result
|
||||
|
||||
|
||||
def driver_sources():
|
||||
return {name: sha(ROOT / name) for name in (
|
||||
'deploy/agent_updater_contract_acceptance.py',
|
||||
'deploy/windows_msi_contract_acceptance.py',
|
||||
'deploy/vm_access_check.py',
|
||||
)}
|
||||
|
||||
|
||||
def remote_source_sha():
|
||||
return hashlib.sha256(inspect.getsource(linux_case).encode()).hexdigest()
|
||||
|
||||
|
||||
def prepare(qualification):
|
||||
setup()
|
||||
require(qualification.absolute() == ROOT / 'devstate/agent-updater/local-qualification.json', 'existing fixed local qualification')
|
||||
old = load(qualification)
|
||||
expected_sources = sources()
|
||||
require(old['profile'] == PROFILE and old['image_id'] == IMAGE and old['target'] == 'x86_64-unknown-linux-gnu'
|
||||
and old['sources'] == expected_sources and old['test_count'] == 9 and old['tests'] == EXPECTED
|
||||
and old['native_cloud_vm_tested'] is False and old['agent_started'] is False and old['activation_implemented'] is False,
|
||||
'original nine local checks only')
|
||||
for label in ('fmt', 'clippy', 'tests'):
|
||||
check = old['checks'][label]
|
||||
require(check['exit_code'] == 0 and check['log_sha256'] == sha(qualification.parent / (label + '.log'))
|
||||
and check['command'][0] == 'docker' and IMAGE in check['command'], 'original local command receipt')
|
||||
common.check_output((qualification.parent / 'tests.log').read_bytes(), 9)
|
||||
item = old['artifact']
|
||||
original = Path(item['path'])
|
||||
expected = ROOT / 'devstate/agent-updater/target/x86_64-unknown-linux-gnu/release/deps/release_contract-2aa0a51dd0c5c97d'
|
||||
meta = original.lstat()
|
||||
require(original == expected and stat.S_ISREG(meta.st_mode) and meta.st_nlink == 1 and meta.st_uid in (0, os.getuid())
|
||||
and item['sha256'] == sha(original) == ARTIFACT_SHA and item['bytes'] == meta.st_size == ARTIFACT_BYTES,
|
||||
'original qualified link, never rebuilt')
|
||||
folder = STATE / ('prepared-' + str(uuid.uuid4()))
|
||||
folder.mkdir(mode=0o700)
|
||||
artifact = folder / 'updater-contracts'
|
||||
fd = os.open(artifact, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, 'wb') as output:
|
||||
output.write(original.read_bytes()); output.flush(); os.fsync(output.fileno())
|
||||
require(sha(artifact) == ARTIFACT_SHA and artifact.read_bytes()[:6] == b'\x7fELF\x02\x01'
|
||||
and artifact.read_bytes()[18:20] == b'\x3e\0', 'same Linux x64 delivery bytes')
|
||||
def inspect_elf(*args):
|
||||
return common.command(['docker', 'run', '--rm', '--network', 'none', '--read-only',
|
||||
'-v', str(artifact) + ':/unit:ro', '--entrypoint', 'readelf', IMAGE, *args, '/unit'], folder).decode()
|
||||
versions = inspect_elf('--version-info')
|
||||
dynamic = inspect_elf('-d')
|
||||
glibc = sorted(set(re.findall(r'GLIBC_(\d+\.\d+)', versions)), key=lambda n: tuple(map(int, n.split('.'))))
|
||||
require(glibc and tuple(map(int, glibc[-1].split('.'))) <= (2, 34), 'known Cloud Linux GLIBC compatibility')
|
||||
manifest = dict(schema_version=1, profile=PROFILE, core_commit=CORE_COMMIT,
|
||||
local_qualification_path=str(qualification.absolute()), local_qualification_sha256=sha(qualification),
|
||||
core_sources=expected_sources, driver_sources=driver_sources(), remote_source_sha256=remote_source_sha(),
|
||||
artifact=dict(path=str(artifact), sha256=sha(artifact), bytes=artifact.stat().st_size,
|
||||
glibc_max=glibc[-1], needed=sorted(re.findall(r'Shared library: \[([^\]]+)\]', dynamic))),
|
||||
test_names=EXPECTED, test_count=9, test_executable_rebuilt=False,
|
||||
local_only=True, native_cloud_vm_tested=False, agent_started=False, activation_implemented=False)
|
||||
save(folder / 'manifest.json', manifest)
|
||||
print(json.dumps(dict(manifest=str(folder / 'manifest.json'), manifest_sha256=sha(folder / 'manifest.json'),
|
||||
tests=9, rebuilt=False, native_executed=False)))
|
||||
|
||||
|
||||
def verified(path, new_case):
|
||||
path = path.absolute()
|
||||
require(path.name == 'manifest.json' and path.parent.parent == STATE
|
||||
and re.fullmatch(r'prepared-[0-9a-f-]{36}', path.parent.name), 'fixed native manifest path')
|
||||
private(path.parent, True)
|
||||
value = load(path)
|
||||
require(value['schema_version'] == 1 and value['profile'] == PROFILE and value['core_commit'] == CORE_COMMIT
|
||||
and value['driver_sources'] == driver_sources() and value['remote_source_sha256'] == remote_source_sha()
|
||||
and value['test_names'] == EXPECTED and value['test_count'] == 9
|
||||
and value['test_executable_rebuilt'] is False and value['native_cloud_vm_tested'] is False,
|
||||
'original reviewed core and native driver')
|
||||
if new_case:
|
||||
require(value['core_sources'] == sources(), 'new native case needs same core')
|
||||
item = value['artifact']
|
||||
require(Path(item['path']) == path.parent / 'updater-contracts'
|
||||
and item['sha256'] == ARTIFACT_SHA and item['bytes'] == ARTIFACT_BYTES, 'fixed delivery tuple')
|
||||
private(item['path'])
|
||||
require(sha(item['path']) == ARTIFACT_SHA and Path(item['path']).stat().st_size == ARTIFACT_BYTES,
|
||||
'retained immutable delivery')
|
||||
return value
|
||||
|
||||
|
||||
def linux_case(p):
|
||||
"""Sent only by the parent to the fixed Cloud Linux bootstrap endpoint."""
|
||||
import resource
|
||||
ident = str(uuid.UUID(p['id']))
|
||||
require(ident == p['id'] and p['profile'] == 'agent-updater-pure-contracts-v1'
|
||||
and p['action'] in ('prepare', 'run', 'read') and p['test_count'] == 9 and len(p['names']) == 9,
|
||||
'fixed pure updater case')
|
||||
require(os.geteuid() == 0 and os.uname().machine == 'x86_64', 'fixed Cloud Linux bootstrap architecture')
|
||||
root = Path('/var/lib/rmm-agent-updater-contracts')
|
||||
def safe(path, directory, executable=False):
|
||||
item = path.lstat()
|
||||
require(item.st_uid == 0 and stat.S_IMODE(item.st_mode) == (0o700 if directory else 0o500 if executable else 0o600)
|
||||
and (stat.S_ISDIR(item.st_mode) if directory else stat.S_ISREG(item.st_mode) and item.st_nlink == 1), 'private unit object')
|
||||
def write(path, value):
|
||||
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, 'w') as output:
|
||||
json.dump(value, output); output.flush(); os.fsync(output.fileno())
|
||||
directory = os.open(path.parent, os.O_DIRECTORY | os.O_RDONLY)
|
||||
try: os.fsync(directory)
|
||||
finally: os.close(directory)
|
||||
def read(path):
|
||||
safe(path, False); require(path.stat().st_size <= 2 * 1024 * 1024, 'bounded original unit receipt')
|
||||
return json.loads(path.read_bytes())
|
||||
os.umask(0o077)
|
||||
case = root / ident
|
||||
if p['action'] == 'read' and not case.exists():
|
||||
return dict(id=ident, status='remote_case_absent')
|
||||
if p['action'] == 'prepare':
|
||||
if not root.exists(): root.mkdir(mode=0o700)
|
||||
safe(root, True); case.mkdir(mode=0o700); write(case / 'intent.json', p)
|
||||
return dict(id=ident, prepared=True)
|
||||
safe(root, True); safe(case, True)
|
||||
original = read(case / 'intent.json')
|
||||
require({k:v for k,v in p.items() if k != 'action'} == {k:v for k,v in original.items() if k != 'action'}, 'exact original unit intent')
|
||||
if p['action'] == 'read':
|
||||
if (case / 'result.json').exists(): return read(case / 'result.json')
|
||||
partial = {name:read(case / name) for name in ('listing.json',) if (case / name).exists()}
|
||||
return dict(id=ident, status='started_without_result' if (case / 'started.json').exists() else 'prepared_without_start', partial=partial)
|
||||
write(case / 'started.json', dict(execute_once=True, manifest_sha256=p['manifest_sha256']))
|
||||
binary = case / 'updater-contracts'; safe(binary, False)
|
||||
require(binary.stat().st_size == p['artifact']['bytes'] == 1544584
|
||||
and hashlib.sha256(binary.read_bytes()).hexdigest() == p['artifact']['sha256'] == 'e3a1a3140da09de2f86055faa45faf7ba05ddddc2d92d23fef3df03a63bf2999', 'exact reviewed updater test executable')
|
||||
binary.chmod(0o500); safe(binary, False, True)
|
||||
temporary = case / 'tmp'; temporary.mkdir(mode=0o700)
|
||||
env = {'PATH':'/usr/bin:/bin', 'LANG':'C.UTF-8', 'LC_ALL':'C.UTF-8', 'TMPDIR':str(temporary), 'RUST_BACKTRACE':'0'}
|
||||
def limits():
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (15, 15))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (1048576, 1048576))
|
||||
resource.setrlimit(resource.RLIMIT_AS, (256 * 1024 * 1024, 256 * 1024 * 1024))
|
||||
require(subprocess.run(['/usr/bin/unshare','--net','--','/usr/bin/true'], env=env, capture_output=True, timeout=3).returncode == 0, 'isolated network namespace required')
|
||||
listed = subprocess.run(['/usr/bin/unshare','--net','--',str(binary),'--list','--format','terse'], cwd=case,
|
||||
env=env, capture_output=True, timeout=5, preexec_fn=limits)
|
||||
require(listed.returncode == 0 and len(listed.stdout) <= 65536 and not listed.stderr, 'bounded actual test list')
|
||||
lines = listed.stdout.decode().splitlines()
|
||||
require(all(not line.strip() or __import__('re').fullmatch(r'[A-Za-z0-9_:]+: test', line) for line in lines), 'terse fixed test names only')
|
||||
actual = sorted(line[:-6] for line in lines if line.endswith(': test'))
|
||||
write(case / 'listing.json', dict(names=actual, sha256=p['artifact']['sha256'], exit_code=listed.returncode))
|
||||
require(actual == p['names'] and len(set(actual)) == 9, 'all nine exact names before test execution')
|
||||
write(case / 'tests-started.json', dict(execute_once=True))
|
||||
with (case / 'stdout.bin').open('xb') as output, (case / 'stderr.bin').open('xb') as errors:
|
||||
timed = False
|
||||
try:
|
||||
code = subprocess.run(['/usr/bin/unshare','--net','--',str(binary),'--test-threads=1'], cwd=case, env=env,
|
||||
stdin=subprocess.DEVNULL, stdout=output, stderr=errors, timeout=20, preexec_fn=limits).returncode
|
||||
except subprocess.TimeoutExpired: timed = True; code = None
|
||||
output.flush(); errors.flush(); os.fsync(output.fileno()); os.fsync(errors.fileno())
|
||||
raw, errors = [(case / name).read_bytes() for name in ('stdout.bin', 'stderr.bin')]
|
||||
require(len(raw) <= 1048576 and len(errors) <= 1048576, 'bounded original native output')
|
||||
result = dict(id=ident, profile=p['profile'], platform='linux', manifest_sha256=p['manifest_sha256'],
|
||||
runner_sha256=p['runner_sha256'], remote_source_sha256=p['remote_source_sha256'], core_commit=p['core_commit'],
|
||||
artifact=p['artifact'], names=actual, exit_code=code, timed_out=timed,
|
||||
stdout_b64=base64.b64encode(raw).decode(), stderr_b64=base64.b64encode(errors).decode(),
|
||||
network_namespace=True, environment_cleared=True, list_before_run=True,
|
||||
product_state_writes=False, rmm_jobs=False, activation_implemented=False)
|
||||
write(case / 'result.json', result)
|
||||
return result
|
||||
|
||||
|
||||
def rpc(prefix, payload, folder):
|
||||
code = 'import os,json,subprocess,base64,hashlib,uuid\nfrom pathlib import Path\nimport stat\n'
|
||||
code += inspect.getsource(common.require) + '\n' + inspect.getsource(linux_case)
|
||||
code += '\ntry:\n print(json.dumps(linux_case(' + repr(payload) + ')))\nexcept Exception as e:\n print(json.dumps(dict(bootstrap_error=True,error_class=type(e).__name__)))\n'
|
||||
return common.transport(prefix, 'python3 -', code.encode(), folder)
|
||||
|
||||
|
||||
def check_result(value, payload):
|
||||
for name in ('id', 'profile', 'manifest_sha256', 'runner_sha256', 'remote_source_sha256', 'core_commit', 'artifact'):
|
||||
require(value.get(name) == payload[name], 'complete native original tuple')
|
||||
require(value.get('platform') == 'linux' and value.get('names') == EXPECTED and value.get('exit_code') == 0
|
||||
and value.get('timed_out') is False and value.get('product_state_writes') is False and value.get('rmm_jobs') is False
|
||||
and value.get('activation_implemented') is False and value.get('network_namespace') is True
|
||||
and value.get('environment_cleared') is True and value.get('list_before_run') is True, 'all pure native constraints')
|
||||
raw = base64.b64decode(value['stdout_b64'], validate=True)
|
||||
errors = base64.b64decode(value['stderr_b64'], validate=True)
|
||||
require(len(raw) <= 1048576 and not errors, 'bounded clean native output')
|
||||
common.check_output(raw, 9)
|
||||
|
||||
|
||||
def execute(path, access_root, case_id):
|
||||
is_readback = case_id is not None
|
||||
setup(); path = path.absolute(); manifest = verified(path, new_case=not is_readback)
|
||||
# This reads access configuration only when the parent explicitly runs native.
|
||||
prefix = common.access_settings(access_root, 'linux')
|
||||
if case_id is None:
|
||||
case_id = str(uuid.uuid4()); folder = STATE / case_id; folder.mkdir(mode=0o700)
|
||||
payload = dict(id=case_id, profile=PROFILE, manifest_sha256=sha(path), runner_sha256=sha(__file__),
|
||||
remote_source_sha256=remote_source_sha(), core_commit=CORE_COMMIT, test_count=9, names=EXPECTED,
|
||||
artifact={name:manifest['artifact'][name] for name in ('sha256', 'bytes')})
|
||||
save(folder / 'intent.json', dict(payload=payload, manifest_path=str(path), access_root=str(access_root)))
|
||||
save(folder / 'started-local.json', dict(no_retry=True))
|
||||
else:
|
||||
require(str(uuid.UUID(case_id)) == case_id, 'original canonical case')
|
||||
folder = STATE / case_id; private(folder, True)
|
||||
original = load(folder / 'intent.json'); load(folder / 'started-local.json')
|
||||
require(original['manifest_path'] == str(path) and original['access_root'] == str(access_root), 'same-case access and manifest')
|
||||
payload = original['payload']
|
||||
require(payload['id'] == case_id and payload['runner_sha256'] == sha(__file__)
|
||||
and payload['manifest_sha256'] == sha(path) and payload['remote_source_sha256'] == remote_source_sha(), 'unchanged same-case runner')
|
||||
if not case_id or not (folder / 'intent.json').exists():
|
||||
raise RuntimeError('missing create-once local intent')
|
||||
# Readback never resumes upload or run, even after a lost prepare reply.
|
||||
if not is_readback:
|
||||
prepared = rpc(prefix, dict(payload, action='prepare'), folder)
|
||||
save(folder / 'prepared-remote.json', prepared)
|
||||
require(prepared == dict(id=case_id, prepared=True), 'fresh fixed remote scratch case')
|
||||
common.upload(prefix, manifest['artifact'], REMOTE + '/' + case_id + '/updater-contracts', folder, 'updater-contracts')
|
||||
result = rpc(prefix, dict(payload, action='read' if is_readback else 'run'), folder)
|
||||
save(folder / ('readback-' + str(uuid.uuid4()) + '.json'), result)
|
||||
if result.get('status'):
|
||||
require(is_readback and result.get('id') == case_id and result['status'] in ('remote_case_absent','prepared_without_start','started_without_result'), 'read-only original partial status')
|
||||
print(json.dumps(dict(verified=False, case_id=case_id, status=result['status'], retry_allowed=False))); return
|
||||
check_result(result, payload)
|
||||
proof = dict(verified=True, core_commit=CORE_COMMIT, manifest_sha256=sha(path), case_id=case_id, native=result)
|
||||
proof_path = path.parent / 'linux-passed.json'
|
||||
if proof_path.exists(): require(load(proof_path) == proof, 'preserve original native proof')
|
||||
else: save(proof_path, proof)
|
||||
print(json.dumps(dict(verified=True, case_id=case_id, tests=9, proof=str(proof_path), proof_sha256=sha(proof_path), activation_tested=False)))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('prepare-local','linux','readback-linux'))
|
||||
parser.add_argument('--qualification', type=Path)
|
||||
parser.add_argument('--manifest', type=Path)
|
||||
parser.add_argument('--case')
|
||||
parser.add_argument('--access-root', type=Path)
|
||||
parser.add_argument('--execute-native-tests', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if args.action == 'prepare-local':
|
||||
require(args.qualification and not (args.manifest or args.case or args.access_root or args.execute_native_tests), 'prepare packages only existing local unit artifact')
|
||||
prepare(args.qualification)
|
||||
else:
|
||||
require(args.manifest and args.access_root and not args.qualification, 'explicit original manifest and access root')
|
||||
if args.action == 'linux': require(args.execute_native_tests and not args.case, 'explicit new native unit invocation')
|
||||
else: require(args.case and not args.execute_native_tests, 'readback never executes tests')
|
||||
execute(args.manifest, args.access_root, args.case)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try: main()
|
||||
except Exception as error:
|
||||
print(json.dumps(dict(error='AGENT_UPDATER_NATIVE_QUALIFICATION_INCOMPLETE', error_class=type(error).__name__, details_suppressed=True)))
|
||||
raise SystemExit(2) from None
|
||||
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Crossbuild/package pure Windows-GNU updater units after actual Cloud Linux9.
|
||||
|
||||
No MSVC, installed agent, LocalService/SYSTEM, updater Apply or VM proof is
|
||||
created by build-local. Only the parent invokes native Windows explicitly.
|
||||
The original Linux driver and its same-case readback remain unchanged.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import uuid
|
||||
|
||||
import agent_updater_contract_acceptance as linux
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STATE = ROOT / 'devstate/agent-updater-windows'
|
||||
PROFILE = 'agent-updater-pure-windows-gnu-v1'
|
||||
TARGET = 'x86_64-pc-windows-gnu'
|
||||
IMAGE = 'sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9'
|
||||
LINUX_PROOF = ROOT / 'devstate/agent-updater-native/prepared-37b4146d-cbef-431a-aa4a-9827841b8482/linux-passed.json'
|
||||
LINUX_PROOF_SHA = 'bb98bdbd010d23e2905cdffa37b2ad38639b43c770ed2efafd7fa9ac8a22f9d4'
|
||||
LINUX_MANIFEST_SHA = 'a7f0903cea77c36c4d40df5ad486161c10e55bb9a9135629a555ffafd411d521'
|
||||
require, save, load, private, sha = linux.require, linux.save, linux.load, linux.private, linux.sha
|
||||
|
||||
|
||||
def setup():
|
||||
os.umask(0o077)
|
||||
STATE.mkdir(mode=0o700, exist_ok=True)
|
||||
private(STATE, True)
|
||||
|
||||
|
||||
def linux_proof():
|
||||
require(sha(LINUX_PROOF) == LINUX_PROOF_SHA and sha(LINUX_PROOF.parent / 'manifest.json') == LINUX_MANIFEST_SHA,
|
||||
'original actual Cloud Linux proof and manifest')
|
||||
proof = load(LINUX_PROOF)
|
||||
manifest = linux.verified(LINUX_PROOF.parent / 'manifest.json', new_case=False)
|
||||
intent = load(linux.STATE / proof['case_id'] / 'intent.json')
|
||||
require(proof['verified'] is True and proof['core_commit'] == linux.CORE_COMMIT
|
||||
and proof['manifest_sha256'] == LINUX_MANIFEST_SHA and proof['case_id'] == '420d53da-6e34-4e04-b871-0b571d8d3fc4'
|
||||
and manifest['test_names'] == linux.EXPECTED, 'actual Linux-first original nine units')
|
||||
linux.check_result(proof['native'], intent['payload'])
|
||||
return manifest
|
||||
|
||||
|
||||
def driver_sources():
|
||||
result = linux.driver_sources()
|
||||
result['deploy/agent_updater_windows_contracts.py'] = sha(__file__)
|
||||
return result
|
||||
|
||||
|
||||
def script_sha():
|
||||
return hashlib.sha256(WINDOWS.encode()).hexdigest()
|
||||
|
||||
|
||||
def build():
|
||||
setup(); linux_proof()
|
||||
before = linux.sources()
|
||||
folder = STATE / ('build-' + str(uuid.uuid4())); folder.mkdir(mode=0o700)
|
||||
target = STATE / 'target'; target.mkdir(mode=0o700, exist_ok=True); private(target, True)
|
||||
save(folder / 'intent.json', dict(core_commit=linux.CORE_COMMIT, core_sources=before,
|
||||
driver_sources=driver_sources(), windows_script_sha256=script_sha(), target=TARGET, image=IMAGE,
|
||||
linux_proof_sha256=LINUX_PROOF_SHA, expected_names=linux.EXPECTED, no_vm=True, no_apply=True))
|
||||
common = ['docker','run','--rm','--network','none','--read-only','--tmpfs','/tmp:rw,size=512m',
|
||||
'-v',str(ROOT)+':/src:ro','-v',str(target)+':/target','-e','CARGO_TARGET_DIR=/target',
|
||||
'--entrypoint','cargo',IMAGE]
|
||||
options = ['--manifest-path','/src/Cargo.toml','--locked','--offline','--target',TARGET,'-p','rmm-updater']
|
||||
linux.common.command(common + ['clippy'] + options + ['--all-targets','--','-D','warnings'], folder)
|
||||
raw = linux.common.command(common + ['test'] + options + ['--release','--test','release_contract','--no-run','--message-format=json'], folder)
|
||||
linked = []
|
||||
for line in raw.splitlines():
|
||||
try: value = json.loads(line)
|
||||
except ValueError: continue
|
||||
if value.get('reason') == 'compiler-artifact' and value.get('executable') and value.get('target',{}).get('name') == 'release_contract' and value.get('profile',{}).get('test'):
|
||||
linked.append(target / Path(value['executable']).relative_to('/target'))
|
||||
require(len(linked) == 1 and linked[0].suffix == '.exe', 'exact single Windows GNU unit executable')
|
||||
original = folder / 'updater-contracts.linked.exe'; delivery = folder / 'updater-contracts.exe'
|
||||
for path in (original, delivery):
|
||||
shutil.copyfile(linked[0], path); path.chmod(0o600)
|
||||
original_hash = sha(original)
|
||||
linux.common.command(['docker','run','--rm','--network','none','--read-only','-v',str(folder)+':/artifacts',
|
||||
'--entrypoint','x86_64-w64-mingw32-strip',IMAGE,'--strip-debug','/artifacts/updater-contracts.exe'], folder)
|
||||
require(sha(linked[0]) == sha(original) == original_hash, 'original Windows link retained')
|
||||
data = delivery.read_bytes(); offset = int.from_bytes(data[60:64], 'little')
|
||||
require(data[:2] == b'MZ' and data[offset:offset+6] == b'PE\0\0\x64\x86' and len(data) <= 16 * 1024 * 1024, 'bounded PE x64 test delivery')
|
||||
details = linux.common.command(['docker','run','--rm','--network','none','--read-only','-v',str(delivery)+':/unit:ro',
|
||||
'--entrypoint','x86_64-w64-mingw32-objdump',IMAGE,'-p','/unit'], folder).decode()
|
||||
imports = sorted(set(re.findall(r'DLL Name:\s+(\S+)', details)))
|
||||
allowed = {'advapi32.dll','api-ms-win-core-synch-l1-2-0.dll','bcrypt.dll','bcryptprimitives.dll','crypt32.dll','kernel32.dll','msvcrt.dll','ntdll.dll','userenv.dll','ws2_32.dll'}
|
||||
require(imports and {name.lower() for name in imports}.issubset(allowed), 'only reviewed Windows system DLL imports')
|
||||
require(linux.sources() == before, 'unchanged actual Linux-qualified core')
|
||||
manifest = dict(schema_version=1, profile=PROFILE, target=TARGET, image=IMAGE, core_commit=linux.CORE_COMMIT,
|
||||
core_sources=before, driver_sources=driver_sources(), windows_script_sha256=script_sha(),
|
||||
linux_proof_sha256=LINUX_PROOF_SHA, linux_manifest_sha256=LINUX_MANIFEST_SHA,
|
||||
expected_names=linux.EXPECTED, expected_count=9, windows_list_actual=False,
|
||||
artifact=dict(path=str(delivery),sha256=sha(delivery),bytes=delivery.stat().st_size,imports=imports),
|
||||
linked=dict(path=str(original),sha256=original_hash), debug_stripped_delivery=True,
|
||||
clippy_D_warnings=True, native_windows_executed=False, native_msvc_runtime_qualified=False,
|
||||
agent_started=False, product_state_writes=False, rmm_jobs=False, activation_implemented=False)
|
||||
save(folder / 'manifest.json', manifest)
|
||||
print(json.dumps(dict(manifest=str(folder / 'manifest.json'),manifest_sha256=sha(folder / 'manifest.json'),
|
||||
artifact=manifest['artifact'], windows_tests_expected=9, native_windows_executed=False, msvc_qualified=False)))
|
||||
|
||||
|
||||
def verified(path, new_case):
|
||||
path = path.absolute()
|
||||
require(path.name == 'manifest.json' and path.parent.parent == STATE
|
||||
and re.fullmatch(r'build-[0-9a-f-]{36}',path.parent.name), 'fixed Windows-GNU manifest')
|
||||
private(path.parent, True)
|
||||
value = load(path)
|
||||
require(value['schema_version'] == 1 and value['profile'] == PROFILE and value['target'] == TARGET and value['image'] == IMAGE
|
||||
and value['core_commit'] == linux.CORE_COMMIT and value['driver_sources'] == driver_sources()
|
||||
and value['windows_script_sha256'] == script_sha() and value['linux_proof_sha256'] == LINUX_PROOF_SHA
|
||||
and value['linux_manifest_sha256'] == LINUX_MANIFEST_SHA and value['expected_names'] == linux.EXPECTED
|
||||
and value['expected_count'] == 9 and value['windows_list_actual'] is False and value['clippy_D_warnings'] is True
|
||||
and value['native_windows_executed'] is False and value['native_msvc_runtime_qualified'] is False,
|
||||
'original Linux-first pure unit scope')
|
||||
if new_case:
|
||||
linux_proof(); require(value['core_sources'] == linux.sources(), 'same original qualified core')
|
||||
for field, name in (('artifact','updater-contracts.exe'),('linked','updater-contracts.linked.exe')):
|
||||
item = value[field]; require(Path(item['path']) == path.parent / name, 'fixed retained artifact name')
|
||||
private(item['path']); require(sha(item['path']) == item['sha256'], 'retained exact Windows bytes')
|
||||
require(Path(value['artifact']['path']).stat().st_size == value['artifact']['bytes'], 'exact PE length')
|
||||
return value
|
||||
|
||||
|
||||
WINDOWS = r'''
|
||||
$ErrorActionPreference='Stop'
|
||||
$p=([Console]::In.ReadLine()|ConvertFrom-Json)
|
||||
if([string]([guid]$p.id)-cne$p.id-or$p.profile-cne'agent-updater-pure-windows-gnu-v1'-or$p.action-notin@('prepare','run','read')-or$p.manifest_sha256-notmatch'^[a-f0-9]{64}$'-or$p.linux_proof_sha256-cne'bb98bdbd010d23e2905cdffa37b2ad38639b43c770ed2efafd7fa9ac8a22f9d4'-or$p.names.Count-ne9){throw 'Fixed Linux-first pure updater unit case'}
|
||||
$root='C:\ProgramData\RustDiscoveryBootstrap';$case=Join-Path $root ('agent-updater-contracts-'+$p.id)
|
||||
function Test-RmmUnitPrivate($path,$dir){$i=Get-Item -LiteralPath $path -Force;if($i.PSIsContainer-ne$dir-or($i.Attributes-band[IO.FileAttributes]::ReparsePoint)){throw 'Unit path type'};$a=Get-Acl -LiteralPath $path;if($a.GetOwner([Security.Principal.SecurityIdentifier]).Value-notin@('S-1-5-18','S-1-5-32-544')){throw 'Private unit owner'};$rules=@($a.GetAccessRules($true,$true,[Security.Principal.SecurityIdentifier]));if(!$rules.Count){throw 'Empty private ACL'};foreach($e in $rules){if($e.AccessControlType-ne'Allow'-or$e.IdentityReference.Value-notin@('S-1-5-18','S-1-5-32-544')-or($e.PropagationFlags-band[Security.AccessControl.PropagationFlags]::InheritOnly)){throw 'Broad private unit ACL'}}}
|
||||
function Write-RmmUnitNew($path,$value){$f=[IO.File]::Open($path,[IO.FileMode]::CreateNew,[IO.FileAccess]::Write,[IO.FileShare]::Read);try{$b=[Text.Encoding]::UTF8.GetBytes(($value|ConvertTo-Json -Depth 16 -Compress));$f.Write($b,0,$b.Length);$f.Flush($true)}finally{$f.Dispose()}}
|
||||
function Get-RmmUnitJson($path){Test-RmmUnitPrivate $path $false;if((Get-Item -LiteralPath $path).Length-gt2097152){throw 'Receipt bound'};return(Get-Content -LiteralPath $path -Raw|ConvertFrom-Json)}
|
||||
function Test-RmmUnitIntent($old){foreach($name in @('id','profile','manifest_sha256','runner_sha256','windows_script_sha256','core_commit','linux_proof_sha256')){if($old.$name-cne$p.$name){throw 'Original unit intent'}};if($old.artifact.sha256-cne$p.artifact.sha256-or$old.artifact.bytes-ne$p.artifact.bytes-or(@($old.names)-join"`n")-cne(@($p.names)-join"`n")){throw 'Original artifact/test tuple'}}
|
||||
if($p.action-eq'read'-and!(Test-Path -LiteralPath $case)){@{id=$p.id;status='remote_case_absent'}|ConvertTo-Json -Compress;exit 0}
|
||||
Test-RmmUnitPrivate $root $true
|
||||
if($p.action-eq'prepare'){if(Test-Path -LiteralPath $case){throw 'Existing case retained'};New-Item -ItemType Directory -Path $case|Out-Null;Test-RmmUnitPrivate $case $true;Write-RmmUnitNew (Join-Path $case 'intent.json') $p;@{id=$p.id;prepared=$true}|ConvertTo-Json -Compress;exit 0}
|
||||
Test-RmmUnitPrivate $case $true;Test-RmmUnitIntent (Get-RmmUnitJson (Join-Path $case 'intent.json'))
|
||||
if($p.action-eq'read'){$result=Join-Path $case 'result.json';if(Test-Path -LiteralPath $result){Get-RmmUnitJson $result|ConvertTo-Json -Depth 16 -Compress;exit 0};$status=if(Test-Path -LiteralPath (Join-Path $case 'started.json')){'started_without_result'}else{'prepared_without_start'};$partial=@{};$listed=Join-Path $case 'listing.json';if(Test-Path -LiteralPath $listed){$partial.listing=Get-RmmUnitJson $listed};@{id=$p.id;status=$status;partial=$partial}|ConvertTo-Json -Depth 16 -Compress;exit 0}
|
||||
function Invoke-RmmUnit($exe,[string]$arguments){$s=New-Object Diagnostics.ProcessStartInfo;$s.FileName=$exe;$s.Arguments=$arguments;$s.UseShellExecute=$false;$s.CreateNoWindow=$true;$s.WorkingDirectory=$case;$s.RedirectStandardOutput=$true;$s.RedirectStandardError=$true;$systemRoot=$env:WINDIR;$s.EnvironmentVariables.Clear();$s.EnvironmentVariables['SystemRoot']=$systemRoot;$s.EnvironmentVariables['WINDIR']=$systemRoot;$s.EnvironmentVariables['PATH']=$systemRoot+'\System32';$s.EnvironmentVariables['TEMP']=$script:temp;$s.EnvironmentVariables['TMP']=$script:temp;$s.EnvironmentVariables['RUST_BACKTRACE']='0';$c=New-Object Diagnostics.Process;$c.StartInfo=$s;if(!$c.Start()){throw 'Own unit start'};try{$o=$c.StandardOutput.ReadToEndAsync();$e=$c.StandardError.ReadToEndAsync();$timed=$false;if(!$c.WaitForExit(20000)){$timed=$true;$c.Kill();if(!$c.WaitForExit(2000)){throw 'Own test process did not exit'}};if(!$o.Wait(2000)-or!$e.Wait(2000)-or$o.Result.Length-gt1048576-or$e.Result.Length-gt1048576){throw 'Bounded unit output'};return @{exit_code=$c.ExitCode;timed_out=$timed;stdout_b64=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($o.Result));stderr_b64=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($e.Result))}}finally{$c.Dispose()}}
|
||||
Write-RmmUnitNew (Join-Path $case 'started.json') @{execute_once=$true;manifest_sha256=$p.manifest_sha256}
|
||||
$exe=Join-Path $case 'updater-contracts.exe';Test-RmmUnitPrivate $exe $false
|
||||
if($p.artifact.sha256-notmatch'^[a-f0-9]{64}$'-or(Get-FileHash -LiteralPath $exe -Algorithm SHA256).Hash.ToLowerInvariant()-cne$p.artifact.sha256-or(Get-Item -LiteralPath $exe).Length-ne$p.artifact.bytes-or$p.artifact.bytes-gt16777216){throw 'Exact bounded Windows GNU PE'}
|
||||
$script:temp=Join-Path $case 'tmp';New-Item -ItemType Directory -Path $script:temp|Out-Null;Test-RmmUnitPrivate $script:temp $true
|
||||
$list=Invoke-RmmUnit $exe '--list --format terse';if($list.exit_code-ne0-or$list.timed_out-or$list.stderr_b64){throw 'Actual Windows test list'}
|
||||
$text=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($list.stdout_b64));$unexpected=@($text-split'\r?\n'|Where-Object{$_.Trim()-and$_-notmatch'^[A-Za-z0-9_:]+: test$'});if($unexpected.Count){throw 'Only exact test names'}
|
||||
[string[]]$names=@($text-split'\r?\n'|Where-Object{$_-match': test$'}|ForEach-Object{$_-replace': test$',''});[Array]::Sort($names,[StringComparer]::Ordinal)
|
||||
Write-RmmUnitNew (Join-Path $case 'listing.json') @{names=$names;sha256=$p.artifact.sha256;listed=$list}
|
||||
if($names.Count-ne9-or($names-join"`n")-cne(@($p.names)-join"`n")){throw 'All nine names before tests'}
|
||||
Write-RmmUnitNew (Join-Path $case 'tests-started.json') @{execute_tests_once=$true}
|
||||
$run=Invoke-RmmUnit $exe '--test-threads=1'
|
||||
$value=@{id=$p.id;profile=$p.profile;platform='windows-gnu';manifest_sha256=$p.manifest_sha256;runner_sha256=$p.runner_sha256;windows_script_sha256=$p.windows_script_sha256;core_commit=$p.core_commit;linux_proof_sha256=$p.linux_proof_sha256;artifact=$p.artifact;names=$names;run=$run;list_before_run=$true;environment_cleared=$true;bootstrap_administrator_units=$true;local_service_proof=$false;system_proof=$false;msvc_runtime_qualified=$false;product_state_writes=$false;rmm_jobs=$false;activation_implemented=$false}
|
||||
Write-RmmUnitNew (Join-Path $case 'result.json') $value;$value|ConvertTo-Json -Depth 16 -Compress
|
||||
'''
|
||||
|
||||
|
||||
def rpc(prefix, payload, folder):
|
||||
script = 'try {\n' + WINDOWS + '\n} catch {[pscustomobject]@{bootstrap_error=$true;error_id=$_.FullyQualifiedErrorId;line=$_.InvocationInfo.ScriptLineNumber}|ConvertTo-Json -Compress;exit 1}'
|
||||
command = 'powershell.exe -NoLogo -NoProfile -NonInteractive -EncodedCommand ' + base64.b64encode(script.encode('utf-16le')).decode()
|
||||
return linux.common.transport(prefix, command, (json.dumps(payload) + '\n').encode(), folder)
|
||||
|
||||
|
||||
def check_result(value, payload):
|
||||
for field in ('id','profile','manifest_sha256','runner_sha256','windows_script_sha256','core_commit','linux_proof_sha256','artifact'):
|
||||
require(value.get(field) == payload[field], 'complete original Windows unit tuple')
|
||||
require(value.get('platform') == 'windows-gnu' and value.get('names') == linux.EXPECTED
|
||||
and value.get('list_before_run') is True and value.get('environment_cleared') is True
|
||||
and value.get('bootstrap_administrator_units') is True and value.get('local_service_proof') is False
|
||||
and value.get('system_proof') is False and value.get('msvc_runtime_qualified') is False
|
||||
and value.get('product_state_writes') is False and value.get('rmm_jobs') is False
|
||||
and value.get('activation_implemented') is False, 'pure GNU bootstrap unit scope only')
|
||||
run = value['run']; require(run['exit_code'] == 0 and run['timed_out'] is False, 'native unit completion')
|
||||
raw = base64.b64decode(run['stdout_b64'], validate=True); errors = base64.b64decode(run['stderr_b64'], validate=True)
|
||||
require(len(raw) <= 1048576 and not errors, 'bounded clean original output')
|
||||
linux.common.check_output(raw, 9)
|
||||
|
||||
|
||||
def execute(path, access_root, case_id):
|
||||
setup(); path = path.absolute(); readback = case_id is not None
|
||||
manifest = verified(path, new_case=not readback)
|
||||
prefix = linux.common.access_settings(access_root, 'windows')
|
||||
if not readback:
|
||||
case_id = str(uuid.uuid4()); folder = STATE / case_id; folder.mkdir(mode=0o700)
|
||||
payload = dict(id=case_id, profile=PROFILE, manifest_sha256=sha(path), runner_sha256=sha(__file__),
|
||||
windows_script_sha256=script_sha(), core_commit=linux.CORE_COMMIT, linux_proof_sha256=LINUX_PROOF_SHA,
|
||||
names=linux.EXPECTED, artifact={field:manifest['artifact'][field] for field in ('sha256','bytes')})
|
||||
save(folder / 'intent.json',dict(payload=payload,manifest_path=str(path),access_root=str(access_root)))
|
||||
save(folder / 'started-local.json',dict(no_retry=True))
|
||||
else:
|
||||
require(str(uuid.UUID(case_id)) == case_id, 'canonical original Windows case')
|
||||
folder = STATE / case_id; private(folder,True); original = load(folder / 'intent.json'); load(folder / 'started-local.json')
|
||||
require(original['manifest_path'] == str(path) and original['access_root'] == str(access_root), 'same original manifest/access')
|
||||
payload = original['payload']; require(payload['id'] == case_id and payload['manifest_sha256'] == sha(path)
|
||||
and payload['runner_sha256'] == sha(__file__) and payload['windows_script_sha256'] == script_sha(), 'same original runner')
|
||||
if not readback:
|
||||
prepared = rpc(prefix,dict(payload,action='prepare'),folder); save(folder / 'prepared-remote.json',prepared)
|
||||
require(prepared == dict(id=case_id,prepared=True), 'new fixed private Windows case')
|
||||
linux.common.upload(prefix,manifest['artifact'],'/C:/ProgramData/RustDiscoveryBootstrap/agent-updater-contracts-'+case_id+'/updater-contracts.exe',folder,'updater-contracts')
|
||||
result = rpc(prefix,dict(payload,action='read' if readback else 'run'),folder)
|
||||
save(folder / ('readback-'+str(uuid.uuid4())+'.json'),result)
|
||||
if result.get('status'):
|
||||
require(readback and result.get('id') == case_id and result['status'] in ('remote_case_absent','prepared_without_start','started_without_result'), 'original read-only partial status')
|
||||
print(json.dumps(dict(verified=False,case_id=case_id,status=result['status'],retry_allowed=False))); return
|
||||
check_result(result,payload)
|
||||
proof = dict(verified=True,manifest_sha256=sha(path),linux_proof_sha256=LINUX_PROOF_SHA,case_id=case_id,native=result)
|
||||
destination = path.parent / 'windows-gnu-passed.json'
|
||||
if destination.exists(): require(load(destination) == proof, 'preserve original Windows unit proof')
|
||||
else: save(destination,proof)
|
||||
print(json.dumps(dict(verified=True,case_id=case_id,tests=9,proof=str(destination),proof_sha256=sha(destination),msvc_qualified=False,apply_tested=False)))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action',choices=('build-local','windows','readback-windows'))
|
||||
parser.add_argument('--manifest',type=Path); parser.add_argument('--access-root',type=Path); parser.add_argument('--case')
|
||||
parser.add_argument('--execute-native-tests',action='store_true'); args = parser.parse_args()
|
||||
if args.action == 'build-local':
|
||||
require(not(args.manifest or args.access_root or args.case or args.execute_native_tests),'crossbuild only'); build()
|
||||
else:
|
||||
require(args.manifest and args.access_root,'original Windows unit manifest and access root')
|
||||
if args.action == 'windows': require(args.execute_native_tests and not args.case,'explicit new native units')
|
||||
else: require(args.case and not args.execute_native_tests,'readback never starts tests')
|
||||
execute(args.manifest,args.access_root,args.case)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try: main()
|
||||
except Exception as error:
|
||||
print(json.dumps(dict(error='AGENT_UPDATER_WINDOWS_UNIT_INCOMPLETE',error_class=type(error).__name__,details_suppressed=True)))
|
||||
raise SystemExit(2) from None
|
||||
Reference in New Issue
Block a user